diff --git a/TEAM_MODULE_INVENTORY.md b/TEAM_MODULE_INVENTORY.md new file mode 100644 index 000000000..3251e33a4 --- /dev/null +++ b/TEAM_MODULE_INVENTORY.md @@ -0,0 +1,54 @@ +# Team module inventory + +## RLA +- Source files: `RLA/paper/lib/deep.py`, `RLA/paper/bin/model.py`, `RLA/paper/tests/test_rla.py`. +- Integrated destination files: `tabm_integrated/paper/lib/deep.py`, `tabm_integrated/paper/bin/run_integrated.py`. +- Configs: official baseline configs plus `model.rla_rank`, `model.rla_first_only`, `model.rla_additive`, `model.rla_init`, `model.rla_base_preserve_noise`, `rla_adapter_lr_multiplier`, `rla_extra_paths_freeze_fraction`. +- Scripts: `tabm_integrated/tools/generate_integrated_configs.py`, `tabm_integrated/tools/run_integrated_matrix.sh`, `tabm_integrated/tools/aggregate_integrated_results.py`. +- Notebooks: none required for RLA. +- Result artifacts: `tabm_integrated/paper/exp/integrated/**/rla*`, `tabm_integrated/paper/exp/final_integrated_summary.csv`, `tabm_integrated/paper/exp/final_integrated_audit.csv`. +- Reproduction command: `N_GPUS=16 tabm_integrated/tools/run_integrated_matrix.sh exp/integrated/manifest_sweeps.txt` followed by final selection and final matrix commands. +- Assumptions: RLA rank-r adapters are used only in TabM-style BatchEnsemble layers. +- Known limitations: CF-FISD alignment with RLA combined variants regularizes the first RLA input path (`R[:, :, 0]`). + +## ESAM +- Source files: `ESAM/code/bin/model.py`, `ESAM/code/lib/*`. +- Integrated destination files: `tabm_integrated/paper/bin/run_integrated.py`, `tabm_integrated/paper/lib/*`. +- Configs: `use_esam`, `esam_rho`, `esam_eps`, `esam_adapter_only`, `esam_memberwise`, warmup/start/end epoch flags, diagnostics flags. +- Scripts: same integrated generator, queue launcher, and aggregator. +- Notebooks: none required for ESAM. +- Result artifacts: `tabm_integrated/paper/exp/integrated/**/esam*` and final CSV/report artifacts. +- Reproduction command: generate configs, then run the smoke/sweep/final manifests with `N_GPUS=16`. +- Assumptions: adapter-only ESAM perturbs member-specific TabM adapters, including RLA `R/S` when present. +- Known limitations: diagnostics are disabled by default to keep committed artifacts compact. + +## MFB +- Source files: `MFB/code/ramp_ncl_mfb_tc5_end_to_end.ipynb`. +- Integrated destination files: `tabm_integrated/paper/bin/run_integrated.py`. +- Configs: `model.mfb.enabled`, `mask_mode`, `mask_granularity`, `keep_rate`, `inverted_scaling`, `use_soft_mask`, `mask_strength`, `anchor_fraction`, `warmup_epochs`, `mask_seed`. +- Scripts: same integrated generator, queue launcher, and aggregator. +- Notebooks: source notebook above; integrated runner ports the legacy member-fixed feature-group MFB path used by the requested keep-rate sweep. +- Result artifacts: `tabm_integrated/paper/exp/integrated/**/mfb*` and final CSV/report artifacts. +- Reproduction command: run the smoke/sweep/final manifests with `N_GPUS=16`. +- Assumptions: requested MFB sweep uses the notebook's legacy hard member-fixed feature-group mask, not the RAMP router ablations. +- Known limitations: RAMP router/fidelity variants from the notebook are not part of the requested keep-rate sweep. + +## CF-FISD +- Source files: `cf_fisd_recovered/paper/lib/cf_fisd.py`, `cf_fisd_recovered/paper/bin/cf_fisd_teachers.py`, `cf_fisd_recovered/paper/tools/generate_cf_fisd_configs.py`, teacher importance arrays under `cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/`. +- Integrated destination files: `tabm_integrated/paper/lib/cf_fisd.py`, `tabm_integrated/paper/tools/cf_fisd_teachers.py`, `tabm_integrated/paper/bin/run_integrated.py`. +- Configs: `cf_fisd.lambda`, `cf_fisd.variant`, `cf_fisd.dataset_name`, `cf_fisd.teacher_dir`, `cf_fisd.teacher_names`, optional member groups. +- Scripts: same integrated generator, queue launcher, and aggregator. +- Notebooks: none required for CF-FISD. +- Result artifacts: `tabm_integrated/paper/exp/integrated/**/cf_fisd*` and final CSV/report artifacts. +- Reproduction command: run generated manifests after verifying teacher arrays exist. +- Assumptions: bundled teammate teacher importance arrays are reused for the five TabReD datasets. +- Known limitations: if teacher arrays are regenerated, final configs should point to the regenerated teacher directory. + +## End-to-end reproduction +1. `cd tabm_integrated/paper && /workspace/.venvs/tabm_integrated/bin/python ../tools/generate_integrated_configs.py --stage initial --clean` +2. `N_GPUS=16 tabm_integrated/tools/run_integrated_matrix.sh exp/integrated/manifest_baseline_fidelity.txt` +3. `N_GPUS=16 tabm_integrated/tools/run_integrated_matrix.sh exp/integrated/manifest_smoke.txt` +4. `N_GPUS=16 tabm_integrated/tools/run_integrated_matrix.sh exp/integrated/manifest_sweeps.txt` +5. `/workspace/.venvs/tabm_integrated/bin/python tabm_integrated/tools/aggregate_integrated_results.py --stage select-final` +6. `N_GPUS=16 tabm_integrated/tools/run_integrated_matrix.sh exp/integrated/manifest_final.txt` +7. `/workspace/.venvs/tabm_integrated/bin/python tabm_integrated/tools/aggregate_integrated_results.py --stage final` diff --git a/final_report/.gitignore b/final_report/.gitignore new file mode 100644 index 000000000..65e51b063 --- /dev/null +++ b/final_report/.gitignore @@ -0,0 +1,2 @@ +aaai_authorkit/ +preview_page_*.png diff --git a/final_report/aaai2026.bst b/final_report/aaai2026.bst new file mode 100644 index 000000000..bc73330ee --- /dev/null +++ b/final_report/aaai2026.bst @@ -0,0 +1,1493 @@ +%% +%% This is file `aaai2026.bst', +%% generated with the docstrip utility. +%% +%% The original source files were: +%% +%% merlin.mbs (with options: `head,ay,nat,ed-au,nm-rev,ed-rev,jnrlst,aunm-semi,mcite,mct-1,mct-x3,keyxyr,dt-beg,yr-per,yrp-per,note-yr,atit-u,volp-sp,num-xser,bkpg-x,add-pub,isbn,ppx,ed,xedn,and-com,and-com-ed,etal-xc,nfss,,{}') +%% merlin.mbs (with options: `tail,ay,nat,ed-au,nm-rev,ed-rev,jnrlst,aunm-semi,mcite,mct-1,mct-x3,keyxyr,dt-beg,yr-per,yrp-per,note-yr,atit-u,volp-sp,num-xser,bkpg-x,add-pub,isbn,ppx,ed,xedn,and-com,and-com-ed,etal-xc,nfss,,{}') +%% ---------------------------------------- +%% *** Natbib-compatible implementation of 'aaai' bib style *** +%% + % =============================================================== + % IMPORTANT NOTICE: + % This bibliographic style (bst) file has been generated from one or + % more master bibliographic style (mbs) files, listed above. + % + % This generated file can be redistributed and/or modified under the terms + % of the LaTeX Project Public License Distributed from CTAN + % archives in directory macros/latex/base/lppl.txt; either + % version 1 of the License, or any later version. + % =============================================================== + % Name and version information of the main mbs file: + % \ProvidesFile{merlin.mbs}[2011/11/18 4.33 (PWD, AO, DPC)] + % For use with BibTeX version 0.99a or later + %------------------------------------------------------------------- + % This bibliography style file is intended for texts in ENGLISH + % This is an author-year citation style bibliography. As such, it is + % non-standard LaTeX, and requires a special package file to function properly. + % Such a package is natbib.sty by Patrick W. Daly + % The form of the \bibitem entries is + % \bibitem[Jones et al.(1990)]{key}... + % \bibitem[Jones et al.(1990)Jones, Baker, and Smith]{key}... + % The essential feature is that the label (the part in brackets) consists + % of the author names, as they should appear in the citation, with the year + % in parentheses following. There must be no space before the opening + % parenthesis! + % With natbib v5.3, a full list of authors may also follow the year. + % In natbib.sty, it is possible to define the type of enclosures that is + % really wanted (brackets or parentheses), but in either case, there must + % be parentheses in the label. + % The \cite command functions as follows: + % \citet{key} ==>> Jones et al. (1990) + % \citet*{key} ==>> Jones, Baker, and Smith (1990) + % \citep{key} ==>> (Jones et al., 1990) + % \citep*{key} ==>> (Jones, Baker, and Smith, 1990) + % \citep[chap. 2]{key} ==>> (Jones et al., 1990, chap. 2) + % \citep[e.g.][]{key} ==>> (e.g. Jones et al., 1990) + % \citep[e.g.][p. 32]{key} ==>> (e.g. Jones et al., 1990, p. 32) + % \citeauthor{key} ==>> Jones et al. + % \citeauthor*{key} ==>> Jones, Baker, and Smith + % \citeyear{key} ==>> 1990 + %--------------------------------------------------------------------- + +ENTRY + { address + archivePrefix + author + booktitle + chapter + edition + editor + eid + eprint + howpublished + institution + isbn + journal + key + month + note + number + organization + pages + publisher + school + series + title + type + volume + year + } + {} + { label extra.label sort.label short.list } +INTEGERS { output.state before.all mid.sentence after.sentence after.block } +FUNCTION {init.state.consts} +{ #0 'before.all := + #1 'mid.sentence := + #2 'after.sentence := + #3 'after.block := +} +STRINGS { s t} +FUNCTION {output.nonnull} +{ 's := + output.state mid.sentence = + { ", " * write$ } + { output.state after.block = + { add.period$ write$ + newline$ + "\newblock " write$ + } + { output.state before.all = + 'write$ + { add.period$ " " * write$ } + if$ + } + if$ + mid.sentence 'output.state := + } + if$ + s +} +FUNCTION {output} +{ duplicate$ empty$ + 'pop$ + 'output.nonnull + if$ +} +FUNCTION {output.check} +{ 't := + duplicate$ empty$ + { pop$ "empty " t * " in " * cite$ * warning$ } + 'output.nonnull + if$ +} +FUNCTION {fin.entry} +{ add.period$ + write$ + newline$ +} + +FUNCTION {new.block} +{ output.state before.all = + 'skip$ + { after.block 'output.state := } + if$ +} +FUNCTION {new.sentence} +{ output.state after.block = + 'skip$ + { output.state before.all = + 'skip$ + { after.sentence 'output.state := } + if$ + } + if$ +} +FUNCTION {add.blank} +{ " " * before.all 'output.state := +} + +FUNCTION {date.block} +{ + new.block +} + +FUNCTION {not} +{ { #0 } + { #1 } + if$ +} +FUNCTION {and} +{ 'skip$ + { pop$ #0 } + if$ +} +FUNCTION {or} +{ { pop$ #1 } + 'skip$ + if$ +} +FUNCTION {new.block.checkb} +{ empty$ + swap$ empty$ + and + 'skip$ + 'new.block + if$ +} +FUNCTION {field.or.null} +{ duplicate$ empty$ + { pop$ "" } + 'skip$ + if$ +} +FUNCTION {emphasize} +{ duplicate$ empty$ + { pop$ "" } + { "\emph{" swap$ * "}" * } + if$ +} +FUNCTION {tie.or.space.prefix} +{ duplicate$ text.length$ #3 < + { "~" } + { " " } + if$ + swap$ +} + +FUNCTION {capitalize} +{ "u" change.case$ "t" change.case$ } + +FUNCTION {space.word} +{ " " swap$ * " " * } + % Here are the language-specific definitions for explicit words. + % Each function has a name bbl.xxx where xxx is the English word. + % The language selected here is ENGLISH +FUNCTION {bbl.and} +{ "and"} + +FUNCTION {bbl.etal} +{ "et~al." } + +FUNCTION {bbl.editors} +{ "eds." } + +FUNCTION {bbl.editor} +{ "ed." } + +FUNCTION {bbl.edby} +{ "edited by" } + +FUNCTION {bbl.edition} +{ "edition" } + +FUNCTION {bbl.volume} +{ "volume" } + +FUNCTION {bbl.of} +{ "of" } + +FUNCTION {bbl.number} +{ "number" } + +FUNCTION {bbl.nr} +{ "no." } + +FUNCTION {bbl.in} +{ "in" } + +FUNCTION {bbl.pages} +{ "" } + +FUNCTION {bbl.page} +{ "" } + +FUNCTION {bbl.chapter} +{ "chapter" } + +FUNCTION {bbl.techrep} +{ "Technical Report" } + +FUNCTION {bbl.mthesis} +{ "Master's thesis" } + +FUNCTION {bbl.phdthesis} +{ "Ph.D. thesis" } + +MACRO {jan} {"January"} + +MACRO {feb} {"February"} + +MACRO {mar} {"March"} + +MACRO {apr} {"April"} + +MACRO {may} {"May"} + +MACRO {jun} {"June"} + +MACRO {jul} {"July"} + +MACRO {aug} {"August"} + +MACRO {sep} {"September"} + +MACRO {oct} {"October"} + +MACRO {nov} {"November"} + +MACRO {dec} {"December"} + +MACRO {acmcs} {"ACM Computing Surveys"} + +MACRO {acta} {"Acta Informatica"} + +MACRO {cacm} {"Communications of the ACM"} + +MACRO {ibmjrd} {"IBM Journal of Research and Development"} + +MACRO {ibmsj} {"IBM Systems Journal"} + +MACRO {ieeese} {"IEEE Transactions on Software Engineering"} + +MACRO {ieeetc} {"IEEE Transactions on Computers"} + +MACRO {ieeetcad} + {"IEEE Transactions on Computer-Aided Design of Integrated Circuits"} + +MACRO {ipl} {"Information Processing Letters"} + +MACRO {jacm} {"Journal of the ACM"} + +MACRO {jcss} {"Journal of Computer and System Sciences"} + +MACRO {scp} {"Science of Computer Programming"} + +MACRO {sicomp} {"SIAM Journal on Computing"} + +MACRO {tocs} {"ACM Transactions on Computer Systems"} + +MACRO {tods} {"ACM Transactions on Database Systems"} + +MACRO {tog} {"ACM Transactions on Graphics"} + +MACRO {toms} {"ACM Transactions on Mathematical Software"} + +MACRO {toois} {"ACM Transactions on Office Information Systems"} + +MACRO {toplas} {"ACM Transactions on Programming Languages and Systems"} + +MACRO {tcs} {"Theoretical Computer Science"} +FUNCTION {bibinfo.check} +{ swap$ + duplicate$ missing$ + { + pop$ pop$ + "" + } + { duplicate$ empty$ + { + swap$ pop$ + } + { swap$ + pop$ + } + if$ + } + if$ +} +FUNCTION {bibinfo.warn} +{ swap$ + duplicate$ missing$ + { + swap$ "missing " swap$ * " in " * cite$ * warning$ pop$ + "" + } + { duplicate$ empty$ + { + swap$ "empty " swap$ * " in " * cite$ * warning$ + } + { swap$ + pop$ + } + if$ + } + if$ +} +FUNCTION {format.eprint} +{ eprint duplicate$ empty$ + 'skip$ + { archivePrefix duplicate$ empty$ + 'skip$ + { ":" * swap$ } + if$ + * "." * + } + if$ +} +INTEGERS { nameptr namesleft numnames } + + +STRINGS { bibinfo} + +FUNCTION {format.names} +{ 'bibinfo := + duplicate$ empty$ 'skip$ { + 's := + "" 't := + #1 'nameptr := + s num.names$ 'numnames := + numnames 'namesleft := + { namesleft #0 > } + { s nameptr + "{vv~}{ll}{, f.}{, jj}" + format.name$ + bibinfo bibinfo.check + 't := + nameptr #1 > + { + namesleft #1 > + { "; " * t * } + { + s nameptr "{ll}" format.name$ duplicate$ "others" = + { 't := } + { pop$ } + if$ + ";" * + t "others" = + { + " " * bbl.etal * + } + { + bbl.and + space.word * t * + } + if$ + } + if$ + } + 't + if$ + nameptr #1 + 'nameptr := + namesleft #1 - 'namesleft := + } + while$ + } if$ +} +FUNCTION {format.names.ed} +{ + format.names +} +FUNCTION {format.key} +{ empty$ + { key field.or.null } + { "" } + if$ +} + +FUNCTION {format.authors} +{ author "author" format.names +} +FUNCTION {get.bbl.editor} +{ editor num.names$ #1 > 'bbl.editors 'bbl.editor if$ } + +FUNCTION {format.editors} +{ editor "editor" format.names duplicate$ empty$ 'skip$ + { + "," * + " " * + get.bbl.editor + * + } + if$ +} +FUNCTION {format.isbn} +{ isbn "isbn" bibinfo.check + duplicate$ empty$ 'skip$ + { + new.block + "ISBN " swap$ * + } + if$ +} + +FUNCTION {format.note} +{ + note empty$ + { "" } + { note #1 #1 substring$ + duplicate$ "{" = + 'skip$ + { output.state mid.sentence = + { "l" } + { "u" } + if$ + change.case$ + } + if$ + note #2 global.max$ substring$ * "note" bibinfo.check + } + if$ +} + +FUNCTION {format.title} +{ title + "title" bibinfo.check +} +FUNCTION {format.full.names} +{'s := + "" 't := + #1 'nameptr := + s num.names$ 'numnames := + numnames 'namesleft := + { namesleft #0 > } + { s nameptr + "{vv~}{ll}" format.name$ + 't := + nameptr #1 > + { + namesleft #1 > + { ", " * t * } + { + s nameptr "{ll}" format.name$ duplicate$ "others" = + { 't := } + { pop$ } + if$ + t "others" = + { + " " * bbl.etal * + } + { + numnames #2 > + { "," * } + 'skip$ + if$ + bbl.and + space.word * t * + } + if$ + } + if$ + } + 't + if$ + nameptr #1 + 'nameptr := + namesleft #1 - 'namesleft := + } + while$ +} + +FUNCTION {author.editor.key.full} +{ author empty$ + { editor empty$ + { key empty$ + { cite$ #1 #3 substring$ } + 'key + if$ + } + { editor format.full.names } + if$ + } + { author format.full.names } + if$ +} + +FUNCTION {author.key.full} +{ author empty$ + { key empty$ + { cite$ #1 #3 substring$ } + 'key + if$ + } + { author format.full.names } + if$ +} + +FUNCTION {editor.key.full} +{ editor empty$ + { key empty$ + { cite$ #1 #3 substring$ } + 'key + if$ + } + { editor format.full.names } + if$ +} + +FUNCTION {make.full.names} +{ type$ "book" = + type$ "inbook" = + or + 'author.editor.key.full + { type$ "proceedings" = + 'editor.key.full + 'author.key.full + if$ + } + if$ +} + +FUNCTION {output.bibitem} +{ newline$ + "\bibitem[{" write$ + label write$ + ")" make.full.names duplicate$ short.list = + { pop$ } + { * } + if$ + "}]{" * write$ + cite$ write$ + "}" write$ + newline$ + "" + before.all 'output.state := +} + +FUNCTION {n.dashify} +{ + 't := + "" + { t empty$ not } + { t #1 #1 substring$ "-" = + { t #1 #2 substring$ "--" = not + { "--" * + t #2 global.max$ substring$ 't := + } + { { t #1 #1 substring$ "-" = } + { "-" * + t #2 global.max$ substring$ 't := + } + while$ + } + if$ + } + { t #1 #1 substring$ * + t #2 global.max$ substring$ 't := + } + if$ + } + while$ +} + +FUNCTION {word.in} +{ bbl.in capitalize + " " * } + +FUNCTION {format.date} +{ year "year" bibinfo.check duplicate$ empty$ + { + "empty year in " cite$ * "; set to ????" * warning$ + pop$ "????" + } + 'skip$ + if$ + extra.label * + before.all 'output.state := + after.sentence 'output.state := +} +FUNCTION {format.btitle} +{ title "title" bibinfo.check + duplicate$ empty$ 'skip$ + { + emphasize + } + if$ +} +FUNCTION {either.or.check} +{ empty$ + 'pop$ + { "can't use both " swap$ * " fields in " * cite$ * warning$ } + if$ +} +FUNCTION {format.bvolume} +{ volume empty$ + { "" } + { bbl.volume volume tie.or.space.prefix + "volume" bibinfo.check * * + series "series" bibinfo.check + duplicate$ empty$ 'pop$ + { swap$ bbl.of space.word * swap$ + emphasize * } + if$ + "volume and number" number either.or.check + } + if$ +} +FUNCTION {format.number.series} +{ volume empty$ + { number empty$ + { series field.or.null } + { series empty$ + { number "number" bibinfo.check } + { output.state mid.sentence = + { bbl.number } + { bbl.number capitalize } + if$ + number tie.or.space.prefix "number" bibinfo.check * * + bbl.in space.word * + series "series" bibinfo.check * + } + if$ + } + if$ + } + { "" } + if$ +} + +FUNCTION {format.edition} +{ edition duplicate$ empty$ 'skip$ + { + output.state mid.sentence = + { "l" } + { "t" } + if$ change.case$ + "edition" bibinfo.check + " " * bbl.edition * + } + if$ +} +INTEGERS { multiresult } +FUNCTION {multi.page.check} +{ 't := + #0 'multiresult := + { multiresult not + t empty$ not + and + } + { t #1 #1 substring$ + duplicate$ "-" = + swap$ duplicate$ "," = + swap$ "+" = + or or + { #1 'multiresult := } + { t #2 global.max$ substring$ 't := } + if$ + } + while$ + multiresult +} +FUNCTION {format.pages} +{ pages duplicate$ empty$ 'skip$ + { duplicate$ multi.page.check + { + n.dashify + } + { + } + if$ + "pages" bibinfo.check + } + if$ +} +FUNCTION {format.journal.pages} +{ pages duplicate$ empty$ 'pop$ + { swap$ duplicate$ empty$ + { pop$ pop$ format.pages } + { + ": " * + swap$ + n.dashify + "pages" bibinfo.check + * + } + if$ + } + if$ +} +FUNCTION {format.journal.eid} +{ eid "eid" bibinfo.check + duplicate$ empty$ 'pop$ + { swap$ duplicate$ empty$ 'skip$ + { + ": " * + } + if$ + swap$ * + } + if$ +} +FUNCTION {format.vol.num.pages} +{ volume field.or.null + duplicate$ empty$ 'skip$ + { + "volume" bibinfo.check + } + if$ + number "number" bibinfo.check duplicate$ empty$ 'skip$ + { + swap$ duplicate$ empty$ + { "there's a number but no volume in " cite$ * warning$ } + 'skip$ + if$ + swap$ + "(" swap$ * ")" * + } + if$ * + eid empty$ + { format.journal.pages } + { format.journal.eid } + if$ +} + +FUNCTION {format.chapter.pages} +{ chapter empty$ + 'format.pages + { type empty$ + { bbl.chapter } + { type "l" change.case$ + "type" bibinfo.check + } + if$ + chapter tie.or.space.prefix + "chapter" bibinfo.check + * * + pages empty$ + 'skip$ + { ", " * format.pages * } + if$ + } + if$ +} + +FUNCTION {format.booktitle} +{ + booktitle "booktitle" bibinfo.check + emphasize +} +FUNCTION {format.in.ed.booktitle} +{ format.booktitle duplicate$ empty$ 'skip$ + { + editor "editor" format.names.ed duplicate$ empty$ 'pop$ + { + "," * + " " * + get.bbl.editor + ", " * + * swap$ + * } + if$ + word.in swap$ * + } + if$ +} +FUNCTION {format.thesis.type} +{ type duplicate$ empty$ + 'pop$ + { swap$ pop$ + "t" change.case$ "type" bibinfo.check + } + if$ +} +FUNCTION {format.tr.number} +{ number "number" bibinfo.check + type duplicate$ empty$ + { pop$ bbl.techrep } + 'skip$ + if$ + "type" bibinfo.check + swap$ duplicate$ empty$ + { pop$ "t" change.case$ } + { tie.or.space.prefix * * } + if$ +} +FUNCTION {format.article.crossref} +{ + word.in + " \cite{" * crossref * "}" * +} +FUNCTION {format.book.crossref} +{ volume duplicate$ empty$ + { "empty volume in " cite$ * "'s crossref of " * crossref * warning$ + pop$ word.in + } + { bbl.volume + capitalize + swap$ tie.or.space.prefix "volume" bibinfo.check * * bbl.of space.word * + } + if$ + " \cite{" * crossref * "}" * +} +FUNCTION {format.incoll.inproc.crossref} +{ + word.in + " \cite{" * crossref * "}" * +} +FUNCTION {format.org.or.pub} +{ 't := + "" + address empty$ t empty$ and + 'skip$ + { + address "address" bibinfo.check * + t empty$ + 'skip$ + { address empty$ + 'skip$ + { ": " * } + if$ + t * + } + if$ + } + if$ +} +FUNCTION {format.publisher.address} +{ publisher "publisher" bibinfo.warn format.org.or.pub +} + +FUNCTION {format.organization.address} +{ organization "organization" bibinfo.check format.org.or.pub +} + +FUNCTION {article} +{ output.bibitem + format.authors "author" output.check + author format.key output + format.date "year" output.check + date.block + format.title "title" output.check + new.block + crossref missing$ + { + journal + "journal" bibinfo.check + emphasize + "journal" output.check + format.vol.num.pages output + } + { format.article.crossref output.nonnull + format.pages output + } + if$ + new.block + format.note output + fin.entry +} +FUNCTION {book} +{ output.bibitem + author empty$ + { format.editors "author and editor" output.check + editor format.key output + } + { format.authors output.nonnull + crossref missing$ + { "author and editor" editor either.or.check } + 'skip$ + if$ + } + if$ + format.date "year" output.check + date.block + format.btitle "title" output.check + crossref missing$ + { format.bvolume output + new.block + format.number.series output + new.sentence + format.publisher.address output + } + { + new.block + format.book.crossref output.nonnull + } + if$ + format.edition output + format.isbn output + new.block + format.note output + fin.entry +} +FUNCTION {booklet} +{ output.bibitem + format.authors output + author format.key output + format.date "year" output.check + date.block + format.title "title" output.check + new.block + howpublished "howpublished" bibinfo.check output + address "address" bibinfo.check output + format.isbn output + new.block + format.note output + fin.entry +} + +FUNCTION {inbook} +{ output.bibitem + author empty$ + { format.editors "author and editor" output.check + editor format.key output + } + { format.authors output.nonnull + crossref missing$ + { "author and editor" editor either.or.check } + 'skip$ + if$ + } + if$ + format.date "year" output.check + date.block + format.btitle "title" output.check + crossref missing$ + { + format.bvolume output + format.chapter.pages "chapter and pages" output.check + new.block + format.number.series output + new.sentence + format.publisher.address output + } + { + format.chapter.pages "chapter and pages" output.check + new.block + format.book.crossref output.nonnull + } + if$ + format.edition output + crossref missing$ + { format.isbn output } + 'skip$ + if$ + new.block + format.note output + fin.entry +} + +FUNCTION {incollection} +{ output.bibitem + format.authors "author" output.check + author format.key output + format.date "year" output.check + date.block + format.title "title" output.check + new.block + crossref missing$ + { format.in.ed.booktitle "booktitle" output.check + format.bvolume output + format.number.series output + format.chapter.pages output + new.sentence + format.publisher.address output + format.edition output + format.isbn output + } + { format.incoll.inproc.crossref output.nonnull + format.chapter.pages output + } + if$ + new.block + format.note output + fin.entry +} +FUNCTION {inproceedings} +{ output.bibitem + format.authors "author" output.check + author format.key output + format.date "year" output.check + date.block + format.title "title" output.check + new.block + crossref missing$ + { format.in.ed.booktitle "booktitle" output.check + format.bvolume output + format.number.series output + format.pages output + new.sentence + publisher empty$ + { format.organization.address output } + { organization "organization" bibinfo.check output + format.publisher.address output + } + if$ + format.isbn output + } + { format.incoll.inproc.crossref output.nonnull + format.pages output + } + if$ + new.block + format.note output + fin.entry +} +FUNCTION {conference} { inproceedings } +FUNCTION {manual} +{ output.bibitem + format.authors output + author format.key output + format.date "year" output.check + date.block + format.btitle "title" output.check + organization address new.block.checkb + organization "organization" bibinfo.check output + address "address" bibinfo.check output + format.edition output + new.block + format.note output + fin.entry +} + +FUNCTION {mastersthesis} +{ output.bibitem + format.authors "author" output.check + author format.key output + format.date "year" output.check + date.block + format.btitle + "title" output.check + new.block + bbl.mthesis format.thesis.type output.nonnull + school "school" bibinfo.warn output + address "address" bibinfo.check output + new.block + format.note output + fin.entry +} + +FUNCTION {misc} +{ output.bibitem + format.authors output + author format.key output + format.date "year" output.check + date.block + format.title output + new.block + howpublished "howpublished" bibinfo.check output + new.block + format.note output + format.eprint output + fin.entry +} +FUNCTION {phdthesis} +{ output.bibitem + format.authors "author" output.check + author format.key output + format.date "year" output.check + date.block + format.btitle + "title" output.check + new.block + bbl.phdthesis format.thesis.type output.nonnull + school "school" bibinfo.warn output + address "address" bibinfo.check output + new.block + format.note output + fin.entry +} + +FUNCTION {proceedings} +{ output.bibitem + format.editors output + editor format.key output + format.date "year" output.check + date.block + format.btitle "title" output.check + format.bvolume output + format.number.series output + new.sentence + publisher empty$ + { format.organization.address output } + { organization "organization" bibinfo.check output + format.publisher.address output + } + if$ + format.isbn output + new.block + format.note output + fin.entry +} + +FUNCTION {techreport} +{ output.bibitem + format.authors "author" output.check + author format.key output + format.date "year" output.check + date.block + format.title + "title" output.check + new.block + format.tr.number output.nonnull + institution "institution" bibinfo.warn output + address "address" bibinfo.check output + new.block + format.note output + fin.entry +} + +FUNCTION {unpublished} +{ output.bibitem + format.authors "author" output.check + author format.key output + format.date "year" output.check + date.block + format.title "title" output.check + new.block + format.note "note" output.check + fin.entry +} + +FUNCTION {default.type} { misc } +READ +FUNCTION {sortify} +{ purify$ + "l" change.case$ +} +INTEGERS { len } +FUNCTION {chop.word} +{ 's := + 'len := + s #1 len substring$ = + { s len #1 + global.max$ substring$ } + 's + if$ +} +FUNCTION {format.lab.names} +{'s := + "" 't := + #1 'nameptr := + s num.names$ 'numnames := + numnames 'namesleft := + { namesleft #0 > } + { s nameptr + "{vv~}{ll}" format.name$ + 't := + nameptr #1 > + { + nameptr #2 = + numnames #3 > and + { "others" 't := + #1 'namesleft := } + 'skip$ + if$ + namesleft #1 > + { ", " * t * } + { + s nameptr "{ll}" format.name$ duplicate$ "others" = + { 't := } + { pop$ } + if$ + t "others" = + { + " " * bbl.etal * + } + { + numnames #2 > + { "," * } + 'skip$ + if$ + bbl.and + space.word * t * + } + if$ + } + if$ + } + 't + if$ + nameptr #1 + 'nameptr := + namesleft #1 - 'namesleft := + } + while$ +} + +FUNCTION {author.key.label} +{ author empty$ + { key empty$ + { cite$ #1 #3 substring$ } + 'key + if$ + } + { author format.lab.names } + if$ +} + +FUNCTION {author.editor.key.label} +{ author empty$ + { editor empty$ + { key empty$ + { cite$ #1 #3 substring$ } + 'key + if$ + } + { editor format.lab.names } + if$ + } + { author format.lab.names } + if$ +} + +FUNCTION {editor.key.label} +{ editor empty$ + { key empty$ + { cite$ #1 #3 substring$ } + 'key + if$ + } + { editor format.lab.names } + if$ +} + +FUNCTION {calc.short.authors} +{ type$ "book" = + type$ "inbook" = + or + 'author.editor.key.label + { type$ "proceedings" = + 'editor.key.label + 'author.key.label + if$ + } + if$ + 'short.list := +} + +FUNCTION {calc.label} +{ calc.short.authors + short.list + "(" + * + year duplicate$ empty$ + short.list key field.or.null = or + { pop$ "" } + 'skip$ + if$ + * + 'label := +} + +FUNCTION {sort.format.names} +{ 's := + #1 'nameptr := + "" + s num.names$ 'numnames := + numnames 'namesleft := + { namesleft #0 > } + { s nameptr + "{vv{ } }{ll{ }}{ f{ }}{ jj{ }}" + format.name$ 't := + nameptr #1 > + { + " " * + namesleft #1 = t "others" = and + { "zzzzz" 't := } + 'skip$ + if$ + t sortify * + } + { t sortify * } + if$ + nameptr #1 + 'nameptr := + namesleft #1 - 'namesleft := + } + while$ +} + +FUNCTION {sort.format.title} +{ 't := + "A " #2 + "An " #3 + "The " #4 t chop.word + chop.word + chop.word + sortify + #1 global.max$ substring$ +} +FUNCTION {author.sort} +{ author empty$ + { key empty$ + { "to sort, need author or key in " cite$ * warning$ + "" + } + { key sortify } + if$ + } + { author sort.format.names } + if$ +} +FUNCTION {author.editor.sort} +{ author empty$ + { editor empty$ + { key empty$ + { "to sort, need author, editor, or key in " cite$ * warning$ + "" + } + { key sortify } + if$ + } + { editor sort.format.names } + if$ + } + { author sort.format.names } + if$ +} +FUNCTION {editor.sort} +{ editor empty$ + { key empty$ + { "to sort, need editor or key in " cite$ * warning$ + "" + } + { key sortify } + if$ + } + { editor sort.format.names } + if$ +} +FUNCTION {presort} +{ calc.label + label sortify + " " + * + type$ "book" = + type$ "inbook" = + or + 'author.editor.sort + { type$ "proceedings" = + 'editor.sort + 'author.sort + if$ + } + if$ + #1 entry.max$ substring$ + 'sort.label := + sort.label + * + " " + * + title field.or.null + sort.format.title + * + #1 entry.max$ substring$ + 'sort.key$ := +} + +ITERATE {presort} +SORT +STRINGS { last.label next.extra } +INTEGERS { last.extra.num last.extra.num.extended last.extra.num.blank number.label } +FUNCTION {initialize.extra.label.stuff} +{ #0 int.to.chr$ 'last.label := + "" 'next.extra := + #0 'last.extra.num := + "a" chr.to.int$ #1 - 'last.extra.num.blank := + last.extra.num.blank 'last.extra.num.extended := + #0 'number.label := +} +FUNCTION {forward.pass} +{ last.label label = + { last.extra.num #1 + 'last.extra.num := + last.extra.num "z" chr.to.int$ > + { "a" chr.to.int$ 'last.extra.num := + last.extra.num.extended #1 + 'last.extra.num.extended := + } + 'skip$ + if$ + last.extra.num.extended last.extra.num.blank > + { last.extra.num.extended int.to.chr$ + last.extra.num int.to.chr$ + * 'extra.label := } + { last.extra.num int.to.chr$ 'extra.label := } + if$ + } + { "a" chr.to.int$ 'last.extra.num := + "" 'extra.label := + label 'last.label := + } + if$ + number.label #1 + 'number.label := +} +FUNCTION {reverse.pass} +{ next.extra "b" = + { "a" 'extra.label := } + 'skip$ + if$ + extra.label 'next.extra := + extra.label + duplicate$ empty$ + 'skip$ + { "{\natexlab{" swap$ * "}}" * } + if$ + 'extra.label := + label extra.label * 'label := +} +EXECUTE {initialize.extra.label.stuff} +ITERATE {forward.pass} +REVERSE {reverse.pass} +FUNCTION {bib.sort.order} +{ sort.label + " " + * + year field.or.null sortify + * + " " + * + title field.or.null + sort.format.title + * + #1 entry.max$ substring$ + 'sort.key$ := +} +ITERATE {bib.sort.order} +SORT +FUNCTION {begin.bib} +{ preamble$ empty$ + 'skip$ + { preamble$ write$ newline$ } + if$ + "\begin{thebibliography}{" number.label int.to.str$ * "}" * + write$ newline$ + "\providecommand{\natexlab}[1]{#1}" + write$ newline$ +} +EXECUTE {begin.bib} +EXECUTE {init.state.consts} +ITERATE {call.type$} +FUNCTION {end.bib} +{ newline$ + "\end{thebibliography}" write$ newline$ +} +EXECUTE {end.bib} +%% End of customized bst file +%% +%% End of file `aaai2026.bst'. diff --git a/final_report/aaai2026.sty b/final_report/aaai2026.sty new file mode 100644 index 000000000..989b76119 --- /dev/null +++ b/final_report/aaai2026.sty @@ -0,0 +1,318 @@ +\NeedsTeXFormat{LaTeX2e}% +\ProvidesPackage{aaai2026}[2026/06/17 AAAI 2026 Submission format]% +\def\year{2026}% +\typeout{Conference Style for AAAI for LaTeX 2e -- version for submission}% +% +\def\copyright@on{T} +\def\showauthors@on{T} +\def\nocopyright{\gdef\copyright@on{}} % Copyright notice is required for camera-ready only. +\DeclareOption{submission}{% + \gdef\copyright@on{}% + \gdef\showauthors@on{}% + \long\gdef\pdfinfo #1{\relax}% +}% +\DeclareOption{draft}{% + \gdef\copyright@on{}% +}% +\ProcessOptions\relax% +% WARNING: IF YOU ARE USING THIS STYLE SHEET FOR AN AAAI PUBLICATION, YOU +% MAY NOT MODIFY IT FOR ANY REASON. MODIFICATIONS (IN YOUR SOURCE +% OR IN THIS STYLE SHEET WILL RESULT IN REJECTION OF YOUR PAPER). +% +% WARNING: This style is NOT guaranteed to work. It is provided in the +% hope that it might make the preparation of papers easier, but this style +% file is provided "as is" without warranty of any kind, either express or +% implied, including but not limited to the implied warranties of +% merchantability, fitness for a particular purpose, or noninfringement. +% You use this style file at your own risk. Standard disclaimers apply. +% There are undoubtably bugs in this style. If you would like to submit +% bug fixes, improvements, etc. please let us know. Please use the contact form +% at www.aaai.org. +% +% Do not use this file unless you are an experienced LaTeX user. +% +% PHYSICAL PAGE LAYOUT +\setlength\topmargin{-0.25in} \setlength\oddsidemargin{-0.25in} +\setlength\textheight{9.0in} \setlength\textwidth{7.0in} +\setlength\columnsep{0.375in} \newlength\titlebox \setlength\titlebox{2.25in} +\setlength\headheight{0pt} \setlength\headsep{0pt} +%\setlength\footheight{0pt} \setlength\footskip{0pt} +\thispagestyle{empty} \pagestyle{empty} +\flushbottom \twocolumn \sloppy +% We're never going to need a table of contents, so just flush it to +% save space --- suggested by drstrip@sandia-2 +\def\addcontentsline#1#2#3{} +% gf: PRINT COPYRIGHT NOTICE +\def\copyright@year{\number\year} +\def\copyright@text{Copyright \copyright\space \copyright@year, +Association for the Advancement of Artificial Intelligence (www.aaai.org). +All rights reserved.} +\def\copyrighttext#1{\gdef\copyright@on{T}\gdef\copyright@text{#1}} +\def\copyrightyear#1{\gdef\copyright@on{T}\gdef\copyright@year{#1}} +% gf: End changes for copyright notice (used in \maketitle, below) +% Title stuff, taken from deproc. +% +\def\maketitle{% + \par% + \begingroup % to make the footnote style local to the title + \def\thefootnote{\fnsymbol{footnote}} + \twocolumn[\@maketitle] \@thanks% + \endgroup% + % Insert copyright slug unless turned off + \if T\copyright@on\insert\footins{\noindent\footnotesize\copyright@text}\fi% + % + \setcounter{footnote}{0}% + \let\maketitle\relax% + \let\@maketitle\relax% + \gdef\@thanks{}% + \gdef\@author{}% + \gdef\@title{}% + \let\thanks\relax% +}% +\long\gdef\affiliations #1{ \def \affiliations_{\if T\showauthors@on#1\fi}}% +% +\def\@maketitle{% + \def\theauthors{\if T\showauthors@on\@author\else Anonymous submission\fi} + \newcounter{eqfn}\setcounter{eqfn}{0}% + \newsavebox{\titlearea} + \sbox{\titlearea}{ + \let\footnote\relax\let\thanks\relax% + \setcounter{footnote}{0}% + \def\equalcontrib{% + \ifnum\value{eqfn}=0% + \footnote{These authors contributed equally.}% + \setcounter{eqfn}{\value{footnote}}% + \else% + \footnotemark[\value{eqfn}]% + \fi% + }% + \vbox{% + \hsize\textwidth% + \linewidth\hsize% + \vskip 0.625in minus 0.125in% + \centering% + {\LARGE\bf \@title \par}% + \vskip 0.1in plus 0.5fil minus 0.05in% + {\Large{\textbf{\theauthors\ifhmode\\\fi}}}% + \vskip .2em plus 0.25fil% + {\normalsize \affiliations_\ifhmode\\\fi}% + \vskip 1em plus 2fil% + }% + }% +% + \newlength\actualheight% + \settoheight{\actualheight}{\usebox{\titlearea}}% + \ifdim\actualheight>\titlebox% + \setlength{\titlebox}{\actualheight}% + \fi% +% + \vbox to \titlebox {% + \let\footnote\thanks\relax% + \setcounter{footnote}{0}% + \def\equalcontrib{% + \ifnum\value{eqfn}=0% + \footnote{These authors contributed equally.}% + \setcounter{eqfn}{\value{footnote}}% + \else% + \footnotemark[\value{eqfn}]% + \fi% + }% + \hsize\textwidth% + \linewidth\hsize% + \vskip 0.625in minus 0.125in% + \centering% + {\LARGE\bf \@title \par}% + \vskip 0.1in plus 0.5fil minus 0.05in% + {\Large{\textbf{\theauthors\ifhmode\\\fi}}}% + \vskip .2em plus 0.25fil% + {\normalsize \affiliations_\ifhmode\\\fi}% + \vskip 1em plus 2fil% + }% +}% +% +\renewenvironment{abstract}{% + \centerline{\bf Abstract}% + \vspace{0.5ex}% + \setlength{\leftmargini}{10pt}% + \begin{quote}% + \small% +}{% + \par% + \end{quote}% + \vskip 1ex% +}% +\newenvironment{links}{% + \newcommand{\link}[2]{\par\textbf{##1} --- \url{##2}}% + \setlength{\hangindent}{10pt}% + \setlength{\parskip}{2pt}% + \begin{flushleft}% +}{% + \end{flushleft}% + \vskip 1ex% +}% +% jsp added: +\def\pubnote#1{ + \thispagestyle{myheadings}% + \pagestyle{myheadings}% + \markboth{#1}{#1}% + \setlength\headheight{10pt}% + \setlength\headsep{10pt}% +}% +% +% SECTIONS with less space +\def\section{\@startsection {section}{1}{\z@}{-2.0ex plus +-0.5ex minus -.2ex}{3pt plus 2pt minus 1pt}{\Large\bf\centering}} +\def\subsection{\@startsection{subsection}{2}{\z@}{-2.0ex plus +-0.5ex minus -.2ex}{3pt plus 2pt minus 1pt}{\large\bf\raggedright}} +\def\subsubsection{\@startsection{subparagraph}{3}{\z@}{-6pt plus +%%% DIEGO changed: 29/11/2009 +%% 2pt minus 1pt}{-1em}{\normalsize\bf}} +-2pt minus -1pt}{-1em}{\normalsize\bf}} +%%% END changed +\renewcommand\paragraph{\@startsection{paragraph}{4}{\z@}{-6pt plus -2pt minus -1pt}{-1em}{\normalsize\bf}}% +\setcounter{secnumdepth}{0} +% add period to section (but not subsection) numbers, reduce space after +%\renewcommand{\thesection} +% {\arabic{section}.\hskip-0.6em} +%\renewcommand{\thesubsection} +% {\arabic{section}.\arabic{subsection}\hskip-0.6em} +% FOOTNOTES +\footnotesep 6.65pt % +\skip\footins 9pt plus 4pt minus 2pt +\def\footnoterule{\kern-3pt \hrule width 5pc \kern 2.6pt } +\setcounter{footnote}{0} +% LISTS AND PARAGRAPHS +\parindent 10pt +\topsep 4pt plus 1pt minus 2pt +\partopsep 1pt plus 0.5pt minus 0.5pt +\itemsep 0.5pt plus 1pt minus 0.5pt +\parsep 2pt plus 1pt minus 0.5pt +\leftmargin 10pt \leftmargini 13pt \leftmarginii 10pt \leftmarginiii 5pt \leftmarginiv 5pt \leftmarginv 5pt \leftmarginvi 5pt +\labelwidth\leftmargini\advance\labelwidth-\labelsep \labelsep 5pt +\def\@listi{\leftmargin\leftmargini} +\def\@listii{\leftmargin\leftmarginii +\labelwidth\leftmarginii\advance\labelwidth-\labelsep +\topsep 2pt plus 1pt minus 0.5pt +\parsep 1pt plus 0.5pt minus 0.5pt +\itemsep \parsep} +\def\@listiii{\leftmargin\leftmarginiii +\labelwidth\leftmarginiii\advance\labelwidth-\labelsep +\topsep 1pt plus 0.5pt minus 0.5pt +\parsep \z@ +\partopsep 0.5pt plus 0pt minus 0.5pt +\itemsep \topsep} +\def\@listiv{\leftmargin\leftmarginiv +\labelwidth\leftmarginiv\advance\labelwidth-\labelsep} +\def\@listv{\leftmargin\leftmarginv +\labelwidth\leftmarginv\advance\labelwidth-\labelsep} +\def\@listvi{\leftmargin\leftmarginvi +\labelwidth\leftmarginvi\advance\labelwidth-\labelsep} +\abovedisplayskip 7pt plus2pt minus5pt% +\belowdisplayskip \abovedisplayskip +\abovedisplayshortskip 0pt plus3pt% +\belowdisplayshortskip 4pt plus3pt minus3pt% +% Less leading in most fonts (due to the narrow columns) +% The choices were between 1-pt and 1.5-pt leading +\def\normalsize{\@setfontsize\normalsize\@xpt{11}} % 10 point on 11 +\def\small{\@setfontsize\small\@ixpt{10}} % 9 point on 10 +\def\footnotesize{\@setfontsize\footnotesize\@ixpt{10}} % 9 point on 10 +\def\scriptsize{\@setfontsize\scriptsize\@viipt{10}} % 7 point on 8 +\def\tiny{\@setfontsize\tiny\@vipt{7}} % 6 point on 7 +\def\large{\@setfontsize\large\@xipt{12}} % 11 point on 12 +\def\Large{\@setfontsize\Large\@xiipt{14}} % 12 point on 14 +\def\LARGE{\@setfontsize\LARGE\@xivpt{16}} % 14 point on 16 +\def\huge{\@setfontsize\huge\@xviipt{20}} % 17 point on 20 +\def\Huge{\@setfontsize\Huge\@xxpt{23}} % 20 point on 23 + +\RequirePackage{xcolor} % Needed for the reproducibility checklist +\def\isChecklistMainFile{} % Signal the checklist that it's included from the main file + +\AtBeginDocument{% + \@ifpackageloaded{natbib}% + {% + % When natbib is in use, set the proper style and fix a few things + \let\cite\citep + \let\shortcite\citeyearpar + \setcitestyle{aysep={}} + \setlength\bibhang{0pt} + \bibliographystyle{aaai2026} + }{}% + \@ifpackageloaded{hyperref}% + {% + \PackageError{aaai}{You must not use hyperref in AAAI papers.}{You (or one of the packages you imported) are importing the hyperref package, which is forbidden in AAAI papers. You must remove it from the paper to proceed.} + }{}% + \@ifpackageloaded{bbm}% + {% + \PackageError{aaai}{You must not use bbm package in AAAI papers because it introduces Type 3 fonts which are forbidden.}{See https://tex.stackexchange.com/questions/479160/a-replacement-to-mathbbm1-with-type-1-fonts for possible alternatives.} + }{}% + \@ifpackageloaded{authblk}% + {% + \PackageError{aaai}{Package authblk is forbbidden.}{Package authblk is forbbiden. You must find an alternative.} + }{}% + \@ifpackageloaded{balance}% + {% + \PackageError{aaai}{Package balance is forbbidden.}{Package balance is forbbiden. You must find an alternative.} + }{}% + \@ifpackageloaded{CJK}% + {% + \PackageError{aaai}{Package CJK is forbbidden.}{Package CJK is forbbiden. You must find an alternative.} + }{}% + \@ifpackageloaded{flushend}% + {% + \PackageError{aaai}{Package flushend is forbbidden.}{Package flushend is forbbiden. You must find an alternative.} + }{}% + \@ifpackageloaded{fontenc}% + {% + \PackageError{aaai}{Package fontenc is forbbidden.}{Package fontenc is forbbiden. You must find an alternative.} + }{}% + \@ifpackageloaded{fullpage}% + {% + \PackageError{aaai}{Package fullpage is forbbidden.}{Package fullpage is forbbiden. You must find an alternative.} + }{}% + \@ifpackageloaded{geometry}% + {% + \PackageError{aaai}{Package geometry is forbbidden.}{Package geometry is forbbiden. You must find an alternative.} + }{}% + \@ifpackageloaded{grffile}% + {% + \PackageError{aaai}{Package grffile is forbbidden.}{Package grffile is forbbiden. You must find an alternative.} + }{}% + \@ifpackageloaded{navigator}% + {% + \PackageError{aaai}{Package navigator is forbbidden.}{Package navigator is forbbiden. You must find an alternative.} + }{}% + \@ifpackageloaded{savetrees}% + {% + \PackageError{aaai}{Package savetrees is forbbidden.}{Package savetrees is forbbiden. You must find an alternative.} + }{}% + \@ifpackageloaded{setspace}% + {% + \PackageError{aaai}{Package setspace is forbbidden.}{Package setspace is forbbiden. You must find an alternative.} + }{}% + \@ifpackageloaded{stfloats}% + {% + \PackageError{aaai}{Package stfloats is forbbidden.}{Package stfloats is forbbiden. You must find an alternative.} + }{}% + \@ifpackageloaded{tabu}% + {% + \PackageError{aaai}{Package tabu is forbbidden.}{Package tabu is forbbiden. You must find an alternative.} + }{}% + \@ifpackageloaded{titlesec}% + {% + \PackageError{aaai}{Package titlesec is forbbidden.}{Package titlesec is forbbiden. You must find an alternative.} + }{}% + \@ifpackageloaded{tocbibind}% + {% + \PackageError{aaai}{Package tocbibind is forbbidden.}{Package tocbibind is forbbiden. You must find an alternative.} + }{}% + \@ifpackageloaded{ulem}% + {% + \PackageError{aaai}{Package ulem is forbbidden.}{Package ulem is forbbiden. You must find an alternative.} + }{}% + \@ifpackageloaded{wrapfig}% + {% + \PackageError{aaai}{Package wrapfig is forbbidden.}{Package wrapfig is forbbiden. You must find an alternative.} + }{}% +} + +\let\endthebibliography=\endlist diff --git a/final_report/build_report.py b/final_report/build_report.py new file mode 100644 index 000000000..03c9990d3 --- /dev/null +++ b/final_report/build_report.py @@ -0,0 +1,778 @@ +from __future__ import annotations + +import io +import math +import subprocess +import textwrap +from pathlib import Path + +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd + + +ROOT = Path(__file__).resolve().parents[1] +OUT = Path(__file__).resolve().parent +SOURCE_REF = "origin/refactor/tabm-integrated-modules" +SUMMARY_PATH = "tabm_integrated/paper/exp/final_integrated_summary.csv" +REPORT_PATH = "tabm_integrated/FINAL_EXPERIMENT_REPORT.md" + +DATASET_ORDER = [ + "sberbank-housing", + "ecom-offers", + "homesite-insurance", + "cooking-time", + "delivery-eta", +] + +DATASET_LABEL = { + "sberbank-housing": "Sberbank", + "ecom-offers": "Ecom", + "homesite-insurance": "Homesite", + "cooking-time": "Cooking", + "delivery-eta": "Delivery", +} + +TABLE_DATA_LABEL = { + "sberbank-housing": "Sb", + "ecom-offers": "Ec", + "homesite-insurance": "Hs", + "cooking-time": "Ck", + "delivery-eta": "Dl", +} + +VARIANT_LABEL = { + "best_rla_only": "RLA", + "best_esam_only": "ESAM", + "best_mfb_only": "MFB", + "best_cf_fisd_only": "CF-FISD", + "best_combined": "Combined", +} + +STATUS_SCORE = { + "clear_win": 2, + "weak_win": 1, + "tie": 0, + "loss": -1, +} + +STATUS_LABEL = { + "clear_win": "Clear win", + "weak_win": "Weak win", + "tie": "Tie", + "loss": "Loss", + "baseline": "Baseline", +} + +COMBO_LABEL = { + "sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001": "R+M+CF", + "mfb_cf_fisd": "M+CF", + "rla_esam": "R+E", + "deliv_comb_rla_cf": "R+CF", +} + + +def git_show(path: str) -> str: + return subprocess.check_output( + ["git", "show", f"{SOURCE_REF}:{path}"], + cwd=ROOT, + text=True, + ) + + +def load_summary() -> pd.DataFrame: + csv_text = git_show(SUMMARY_PATH) + (OUT / "source_final_integrated_summary.csv").write_text(csv_text) + return pd.read_csv(io.StringIO(csv_text)) + + +def fmt_num(x: float, digits: int = 6) -> str: + if x is None or not math.isfinite(float(x)): + return "--" + x = float(x) + if abs(x) >= 10: + return f"{x:.3f}" + if abs(x) >= 1: + return f"{x:.4f}" + return f"{x:.6f}" + + +def fmt_pm(mean: float, std: float) -> str: + return f"{fmt_num(mean)} $\\pm$ {fmt_num(std)}" + + +def fmt_compact(x: float) -> str: + if x is None or not math.isfinite(float(x)): + return "--" + return f"{float(x):.4f}" + + +def fmt_pct(x: float) -> str: + if x is None or not math.isfinite(float(x)): + return "--" + sign = "+" if x > 0 else "" + return f"{sign}{x:.3f}\\%" + + +def latex_escape(value: object) -> str: + s = str(value) + return ( + s.replace("\\", "\\textbackslash{}") + .replace("&", "\\&") + .replace("%", "\\%") + .replace("$", "\\$") + .replace("#", "\\#") + .replace("_", "\\_") + .replace("{", "\\{") + .replace("}", "\\}") + .replace("~", "\\textasciitilde{}") + .replace("^", "\\textasciicircum{}") + ) + + +def short_inference(mode: str) -> str: + return { + "mean": "mean", + "best-head": "best", + "greedy-heads": "greedy", + }.get(mode, mode) + + +def default_delta_pct(row: pd.Series, base_rows: pd.DataFrame) -> float: + base = base_rows.loc[row["dataset"]] + if row["direction"] == "lower": + delta = float(base["mean"]) - float(row["mean"]) + else: + delta = float(row["mean"]) - float(base["mean"]) + return 100.0 * delta / abs(float(base["mean"])) + + +def make_combined_plot(df: pd.DataFrame) -> None: + base = df[df["variant"] == "baseline_plr"].set_index("dataset") + combined = df[df["variant"] == "best_combined"].set_index("dataset").loc[DATASET_ORDER].reset_index() + matched = combined["percent_delta"].astype(float).to_numpy() + default = np.array([default_delta_pct(row, base) for _, row in combined.iterrows()]) + + x = np.arange(len(DATASET_ORDER)) + width = 0.36 + fig, ax = plt.subplots(figsize=(7.2, 3.0)) + ax.axhline(0, color="#2b2b2b", linewidth=0.8) + ax.bar(x - width / 2, matched, width, label="Matched inference", color="#234f68") + ax.bar(x + width / 2, default, width, label="Default mean baseline", color="#c47d30") + ax.set_xticks(x) + ax.set_xticklabels([DATASET_LABEL[d] for d in DATASET_ORDER], rotation=20, ha="right") + ax.set_ylabel("Improvement over baseline (%)") + ax.set_title("Combined method: matched and default-mean comparisons") + ax.legend(frameon=False, fontsize=8) + ax.grid(axis="y", alpha=0.25) + fig.tight_layout() + fig.savefig(OUT / "fig_combined_comparison.pdf") + fig.savefig(OUT / "fig_combined_comparison.png", dpi=220) + plt.close(fig) + + +def make_status_heatmap(df: pd.DataFrame) -> None: + variants = list(VARIANT_LABEL) + matrix = [] + for variant in variants: + row = [] + rows = df[df["variant"] == variant].set_index("dataset") + for ds in DATASET_ORDER: + row.append(STATUS_SCORE[str(rows.loc[ds, "status"])]) + matrix.append(row) + matrix = np.array(matrix, dtype=float) + + fig, ax = plt.subplots(figsize=(7.2, 2.7)) + cmap = plt.matplotlib.colors.ListedColormap(["#b94b45", "#d7d7d7", "#e0b44d", "#3c7d59"]) + bounds = [-1.5, -0.5, 0.5, 1.5, 2.5] + norm = plt.matplotlib.colors.BoundaryNorm(bounds, cmap.N) + ax.imshow(matrix, cmap=cmap, norm=norm, aspect="auto") + ax.set_xticks(np.arange(len(DATASET_ORDER))) + ax.set_xticklabels([DATASET_LABEL[d] for d in DATASET_ORDER], rotation=20, ha="right") + ax.set_yticks(np.arange(len(variants))) + ax.set_yticklabels([VARIANT_LABEL[v] for v in variants]) + for i, variant in enumerate(variants): + rows = df[df["variant"] == variant].set_index("dataset") + for j, ds in enumerate(DATASET_ORDER): + status = str(rows.loc[ds, "status"]) + txt = {"clear_win": "C", "weak_win": "W", "tie": "T", "loss": "L"}[status] + ax.text(j, i, txt, ha="center", va="center", fontsize=9, color="black") + ax.set_title("Validation-selected module outcomes vs matched baselines") + ax.tick_params(length=0) + fig.tight_layout() + fig.savefig(OUT / "fig_module_heatmap.pdf") + fig.savefig(OUT / "fig_module_heatmap.png", dpi=220) + plt.close(fig) + + +def make_win_count_plot(df: pd.DataFrame) -> None: + variants = list(VARIANT_LABEL) + counts = [] + for variant in variants: + rows = df[df["variant"] == variant] + counts.append(int(rows["status"].isin(["weak_win", "clear_win"]).sum())) + fig, ax = plt.subplots(figsize=(5.8, 2.5)) + ax.bar([VARIANT_LABEL[v] for v in variants], counts, color="#4f6f52") + ax.set_ylim(0, 5.4) + ax.set_ylabel("Datasets improved (of 5)") + ax.set_title("Individual modules are complementary; combined reaches 5/5") + for idx, c in enumerate(counts): + ax.text(idx, c + 0.08, str(c), ha="center", va="bottom", fontsize=9) + ax.grid(axis="y", alpha=0.25) + fig.tight_layout() + fig.savefig(OUT / "fig_win_counts.pdf") + fig.savefig(OUT / "fig_win_counts.png", dpi=220) + plt.close(fig) + + +def make_delta_heatmap(df: pd.DataFrame) -> None: + variants = list(VARIANT_LABEL) + matrix = [] + for variant in variants: + rows = df[df["variant"] == variant].set_index("dataset") + matrix.append([float(rows.loc[ds, "percent_delta"]) for ds in DATASET_ORDER]) + matrix = np.array(matrix) + vmax = max(2.2, float(np.nanmax(np.abs(matrix)))) + fig, ax = plt.subplots(figsize=(7.4, 2.9)) + im = ax.imshow(matrix, cmap="RdYlGn", vmin=-vmax, vmax=vmax, aspect="auto") + ax.set_xticks(np.arange(len(DATASET_ORDER))) + ax.set_xticklabels([DATASET_LABEL[d] for d in DATASET_ORDER], rotation=20, ha="right") + ax.set_yticks(np.arange(len(variants))) + ax.set_yticklabels([VARIANT_LABEL[v] for v in variants]) + for i in range(matrix.shape[0]): + for j in range(matrix.shape[1]): + ax.text(j, i, f"{matrix[i, j]:+.2f}", ha="center", va="center", fontsize=8) + ax.set_title("Matched-inference improvement (%) by module") + ax.tick_params(length=0) + cb = fig.colorbar(im, ax=ax, fraction=0.035, pad=0.02) + cb.set_label("Improvement (%)", fontsize=8) + fig.tight_layout() + fig.savefig(OUT / "fig_module_delta_heatmap.pdf") + fig.savefig(OUT / "fig_module_delta_heatmap.png", dpi=220) + plt.close(fig) + + +def make_protocol_plot() -> None: + fig, ax = plt.subplots(figsize=(7.4, 2.2)) + ax.axis("off") + boxes = [ + ("TabM paper\nPLR configs", 0.08, "#d9e7ef"), + ("Reproduced\nTabM baseline", 0.29, "#e7ead7"), + ("Member-level\nmodules", 0.50, "#efe1cf"), + ("Validation-only\nselection", 0.71, "#ead8e9"), + ("Matched + default\ncomparisons", 0.90, "#dce6da"), + ] + y = 0.52 + for label, x, color in boxes: + ax.text( + x, + y, + label, + ha="center", + va="center", + fontsize=9, + bbox=dict(boxstyle="round,pad=0.35", facecolor=color, edgecolor="#333333", linewidth=0.9), + transform=ax.transAxes, + ) + for (_, x0, _), (_, x1, _) in zip(boxes[:-1], boxes[1:]): + ax.annotate( + "", + xy=(x1 - 0.08, y), + xytext=(x0 + 0.08, y), + arrowprops=dict(arrowstyle="->", linewidth=1.2, color="#333333"), + xycoords=ax.transAxes, + textcoords=ax.transAxes, + ) + ax.text( + 0.5, + 0.13, + "No data-policy changes, no paper-baseline retuning, final rows use three seeds.", + ha="center", + va="center", + fontsize=8, + color="#333333", + transform=ax.transAxes, + ) + fig.tight_layout() + fig.savefig(OUT / "fig_protocol_bridge.pdf") + fig.savefig(OUT / "fig_protocol_bridge.png", dpi=220) + plt.close(fig) + + +def make_method_diagram() -> None: + fig, ax = plt.subplots(figsize=(7.4, 3.1)) + ax.axis("off") + + boxes = [ + ("Input\nfeatures", 0.08, 0.55, "#e9ecef"), + ("MFB\nmember masks", 0.25, 0.55, "#f1dfc9"), + ("PLR\nembeddings", 0.42, 0.55, "#dce8f2"), + ("Shared\nMLP", 0.59, 0.55, "#e4ebd4"), + ("Member\nadapters", 0.76, 0.55, "#ead8e8"), + ("Predictions\nmean/best/greedy", 0.92, 0.55, "#dbe7dc"), + ] + for label, x, y, color in boxes: + ax.text( + x, + y, + label, + ha="center", + va="center", + fontsize=9, + bbox=dict(boxstyle="round,pad=0.38", facecolor=color, edgecolor="#333333", linewidth=0.9), + transform=ax.transAxes, + ) + for (_, x0, y0, _), (_, x1, y1, _) in zip(boxes[:-1], boxes[1:]): + ax.annotate( + "", + xy=(x1 - 0.065, y1), + xytext=(x0 + 0.065, y0), + arrowprops=dict(arrowstyle="->", linewidth=1.2, color="#333333"), + xycoords=ax.transAxes, + textcoords=ax.transAxes, + ) + + annotations = [ + ("ESAM perturbs\nadapter parameters\nduring training", 0.70, 0.20, 0.74, 0.44), + ("RLA changes\nadapter rank", 0.77, 0.86, 0.77, 0.66), + ("CF-FISD aligns\nadapter saliency", 0.90, 0.86, 0.79, 0.66), + ] + for text, tx, ty, px, py in annotations: + ax.annotate( + text, + xy=(px, py), + xytext=(tx, ty), + ha="center", + va="center", + fontsize=8, + bbox=dict(boxstyle="round,pad=0.25", facecolor="#ffffff", edgecolor="#555555", linewidth=0.8), + arrowprops=dict(arrowstyle="->", linewidth=1.0, color="#555555"), + xycoords=ax.transAxes, + textcoords=ax.transAxes, + ) + ax.text( + 0.5, + 0.04, + "All modules are member-level changes around the official TabM-PLR pipeline.", + ha="center", + va="center", + fontsize=8, + transform=ax.transAxes, + ) + fig.tight_layout() + fig.savefig(OUT / "fig_method_diagram.pdf") + fig.savefig(OUT / "fig_method_diagram.png", dpi=220) + plt.close(fig) + + +def combined_table(df: pd.DataFrame) -> str: + base = df[df["variant"] == "baseline_plr"].set_index("dataset") + combined = df[df["variant"] == "best_combined"].set_index("dataset").loc[DATASET_ORDER].reset_index() + lines = [ + "\\begin{table}[!htbp]", + "\\centering", + "\\setlength{\\tabcolsep}{2pt}", + "\\footnotesize", + "\\begin{tabular}{lllccc}", + "\\hline", + "Data & Combo & Inf. & $\\Delta_m$ & $\\Delta_\\mu$ & S \\\\", + "\\hline", + ] + for _, row in combined.iterrows(): + mean_delta = default_delta_pct(row, base) + combo = COMBO_LABEL.get(str(row["source_variant"]), str(row["source_variant"])) + data_metric = f"{TABLE_DATA_LABEL[row['dataset']]}/{row['metric']}" + status_code = {"clear_win": "C", "weak_win": "W", "tie": "T", "loss": "L"}[str(row["status"])] + lines.append( + f"{latex_escape(data_metric)} & " + f"{latex_escape(combo)} & " + f"{latex_escape(short_inference(str(row['inference_mode'])))} & " + f"{fmt_pct(row['percent_delta'])} & " + f"{fmt_pct(mean_delta)} & " + f"{status_code} \\\\" + ) + lines += [ + "\\hline", + "\\end{tabular}", + "\\caption{Final validation-selected combined results. Data includes the metric. Combo abbreviations: R = RLA, E = ESAM, M = MFB, CF = CF-FISD. $\\Delta_m$ is matched-inference improvement; $\\Delta_\\mu$ is improvement over default mean-inference TabM-PLR. S: C = clear win, W = weak win. Mean and standard deviation values are in the released CSV artifact.}", + "\\label{tab:combined}", + "\\end{table}", + ] + return "\n".join(lines) + + +def ablation_summary_table() -> str: + rows = [ + ("Baseline fidelity", "Paper-config TabM-PLR is the reference row; we do not lower or simplify the baseline."), + ("RLA capacity", "Rank, initialization, noise, freeze, and inference sweeps show capacity helps Sb. and Dl., but not every dataset."), + ("ESAM sharpness", "Rho and adapter-only sweeps show ESAM helps Ec., Hs., and Ck."), + ("MFB diversity", "Member mask and keep-rate sweeps show feature diversity helps Ec., Ck., and Dl."), + ("CF-FISD saliency", "Teacher-alignment and lambda sweeps help Sb. and Hs.; Dl. is neutral under matched inference."), + ("Combined subsets", "Compatible module subsets are validation-selected per dataset; the selected combinations improve 5/5 matched."), + ("Inference audit", "Mean, best-head, and greedy-head baselines are compared separately to avoid inflated claims."), + ("Rejected screens", "Auxiliary objectives and k-scaling screens are not used in the final selected rows."), + ] + lines = [ + "\\begin{table}[!htbp]", + "\\centering", + "\\footnotesize", + "\\setlength{\\tabcolsep}{2pt}", + "\\begin{tabular}{p{0.27\\columnwidth}p{0.65\\columnwidth}}", + "\\hline", + "Ablation & What it showed \\\\", + "\\hline", + ] + for name, takeaway in rows: + lines.append(f"{latex_escape(name)} & {latex_escape(takeaway)} \\\\") + lines += [ + "\\hline", + "\\end{tabular}", + "\\caption{Ablation coverage in the main report. The raw grids contain more individual trials; the table lists the ablation families that affect the final conclusion.}", + "\\label{tab:ablations}", + "\\end{table}", + ] + return "\n".join(lines) + + +def paper_alignment_table() -> str: + rows = [ + ("Paper target", "TabM, ICLR 2025"), + ("Baseline", "Official per-dataset TabM-PLR configs"), + ("Data", "Five available TabReD datasets"), + ("Metrics", "RMSE for regression, AUROC for binary tasks"), + ("Kept fixed", "Preprocessing, data policy, tuned optimizer, $k$, patience"), + ("Changed", "Only module-specific adapter/loss/masking flags"), + ("Selection", "Validation-only; report matched and default-mean baselines"), + ] + lines = [ + "\\begin{table}[!htbp]", + "\\centering", + "\\small", + "\\begin{tabular}{p{0.23\\columnwidth}p{0.65\\columnwidth}}", + "\\hline", + "Item & Protocol \\\\", + "\\hline", + ] + for item, protocol in rows: + lines.append(f"{latex_escape(item)} & {protocol} \\\\") + lines += [ + "\\hline", + "\\end{tabular}", + "\\caption{Comparison protocol against the TabM paper implementation. The report compares against a reproduced paper-config TabM-PLR baseline, not a simplified local baseline.}", + "\\label{tab:paper_alignment}", + "\\end{table}", + ] + return "\n".join(lines) + + +def baseline_values_table(df: pd.DataFrame) -> str: + base = df[df["variant"] == "baseline_plr"].set_index("dataset") + combined = df[df["variant"] == "best_combined"].set_index("dataset").loc[DATASET_ORDER].reset_index() + lines = [ + "\\begin{table}[!htbp]", + "\\centering", + "\\footnotesize", + "\\setlength{\\tabcolsep}{2pt}", + "\\begin{tabular}{lcccc}", + "\\hline", + "Data & Default & Matched & Ours & $\\Delta_m$ \\\\", + "\\hline", + ] + for _, row in combined.iterrows(): + ds = row["dataset"] + data_metric = f"{TABLE_DATA_LABEL[ds]}/{row['metric']}" + lines.append( + f"{latex_escape(data_metric)} & " + f"{fmt_compact(base.loc[ds, 'mean'])} & " + f"{fmt_compact(row['matched_baseline_mean'])} & " + f"{fmt_compact(row['mean'])} & " + f"{fmt_pct(row['percent_delta'])} \\\\" + ) + lines += [ + "\\hline", + "\\end{tabular}", + "\\caption{Paper-config baseline comparison. Default is the reproduced default mean-inference TabM-PLR baseline. Matched is TabM-PLR under the same selected inference mode as Ours.}", + "\\label{tab:baseline_values}", + "\\end{table}", + ] + return "\n".join(lines) + + +def module_table(df: pd.DataFrame) -> str: + variants = list(VARIANT_LABEL) + code = {"clear_win": "C", "weak_win": "W", "tie": "T", "loss": "L"} + lines = [ + "\\begin{table}[!htbp]", + "\\centering", + "\\small", + "\\begin{tabular}{lccccc}", + "\\hline", + "Method & Sb. & Ec. & Hs. & Ck. & Dl. \\\\", + "\\hline", + ] + for variant in variants: + rows = df[df["variant"] == variant].set_index("dataset") + vals = [code[str(rows.loc[ds, "status"])] for ds in DATASET_ORDER] + lines.append(f"{latex_escape(VARIANT_LABEL[variant])} & " + " & ".join(vals) + " \\\\") + lines += [ + "\\hline", + "\\end{tabular}", + "\\caption{Matched-baseline outcome matrix. C = clear win, W = weak win, T = tie, L = loss. Dataset abbreviations follow Table~\\ref{tab:combined}.}", + "\\label{tab:matrix}", + "\\end{table}", + ] + return "\n".join(lines) + + +def make_tex(df: pd.DataFrame) -> str: + combined = combined_table(df) + baseline_values = baseline_values_table(df) + ablations = ablation_summary_table() + modules = module_table(df) + report_md = git_show(REPORT_PATH) + (OUT / "source_final_experiment_report.md").write_text(report_md) + + return textwrap.dedent( + rf""" + % Auto-generated by final_report/build_report.py. + \documentclass[letterpaper]{{article}} + \usepackage{{aaai2026}} + \nocopyright + \usepackage{{times}} + \usepackage{{helvet}} + \usepackage{{courier}} + \usepackage[hyphens]{{url}} + \usepackage{{graphicx}} + \urlstyle{{rm}} + \def\UrlFont{{\rm}} + \usepackage{{natbib}} + \usepackage{{caption}} + \frenchspacing + \setlength{{\pdfpagewidth}}{{8.5in}} + \setlength{{\pdfpageheight}}{{11in}} + \providecommand{{\pdfinfo}}[1]{{}} + \pdfinfo{{/TemplateVersion (2026.1)}} + \setcounter{{secnumdepth}}{{0}} + + \title{{Complementary Member-Level Regularization for TabM on TabReD}} + \author{{Prithvi Nishal (G2504965H), Pentamsetty Sai Harshita (G2503340A),\\ + Nath Simerjit Kaur (G2507742D), Abhipray Chavan (G2504327J)}} + \affiliations{{Nanyang Technological University}} + + \begin{{document}} + \maketitle + + \begin{{abstract}} + TabM is a recent tabular deep learning baseline built around parameter-efficient ensembling. We test four concrete member-level questions on five TabReD datasets: whether TabM benefits from more adapter capacity, flatter adapter optima, fixed member-level feature diversity, or structured adapter saliency. Our implementation keeps the official TabM-PLR per-dataset pipeline and adds four corresponding modifications: rank-adaptive low-rank adapters (RLA), adapter-focused sharpness-aware optimization (ESAM), member-fixed feature bagging (MFB), and teacher-importance distillation over adapter feature saliency (CF-FISD). Individually, no module dominates all datasets. Under validation-selected matched inference, a per-dataset set of compatible module combinations improves over the corresponding TabM-PLR baseline on all five datasets. The result is intentionally scoped: default mean-inference TabM-PLR remains better on some datasets, so the main finding is complementarity under matched inference, not a replacement for the paper baseline. + \end{{abstract}} + + \section{{Introduction}} + Tabular data remains a difficult setting for deep learning because tree models and well-tuned MLPs are hard baselines. TabM~\citep{{gorishniy2025tabm}} addresses this by producing multiple predictions from one parameter-efficient MLP ensemble based on BatchEnsemble-style member adapters~\citep{{wen2020batchensemble}}. The paper shows that the members are weak individually but useful collectively; this makes the internal member structure a concrete part of the model to test. Our project asks whether that structure can be improved by shaping capacity, optimization, feature exposure, and feature saliency at the member level. + + We use the project option of improving a post-2019 conference paper with its public codebase. The target paper is TabM, an ICLR 2025 paper, and the benchmark is TabReD~\citep{{rubachev2024tabred}}, a tabular benchmark with realistic splits and heterogeneous tasks. PLR in this report means the PiecewiseLinearEmbeddings numerical embedding setup used by the TabM paper configs. The contribution is not a new dataset or a lowered baseline. Instead, the report evaluates four modifications on top of the same TabM-PLR per-dataset configs. + + The main result has a narrow scope. The validation-selected combined method improves over matched-inference TabM-PLR on all five selected TabReD datasets. This is not the same as beating the default mean-inference TabM baseline everywhere. We report both comparisons to avoid artificially lowering the baseline. + + \section{{Relationship to the TabM Paper}} + The paper comparison in this report is against the official TabM-PLR implementation path. We do not replace TabM with a simplified MLP, retune the baseline downward, or change the per-dataset data policies. The official paper pipeline uses TabM with PiecewiseLinearEmbeddings, tuned optimizer settings, dataset-specific policies, and $k=32$ members for these TabReD-style experiments. Our code keeps that pipeline and adds module flags around the member adapters. + + Table~\ref{{tab:paper_alignment}} summarizes the alignment. The important distinction is that the released paper used a broader evaluation budget, while our final confirmation run uses three seeds for every final row. Therefore, our result should be read as a controlled project reproduction and extension of the TabM paper configuration, not as a claim that we have replaced the full published leaderboard. + + {paper_alignment_table()} + + \begin{{figure}}[!htbp] + \centering + \includegraphics[width=\columnwidth]{{fig_protocol_bridge.pdf}} + \caption{{How the experiment connects the TabM paper implementation to our modified system.}} + \label{{fig:protocol}} + \end{{figure}} + + \section{{Methods}} + \textbf{{TabM baseline.}} TabM trains an MLP backbone that emits $k$ member predictions using parameter-efficient per-member multiplicative adapters. At inference, the default prediction is the mean over members, while best-head and greedy-head modes select heads or subsets using validation scores. Our baseline is the official TabM-PLR configuration for each dataset: tuned optimizer, architecture, embeddings, data policy, $k$, patience, and dataset-specific preprocessing are preserved. + + \textbf{{RLA.}} Rank-adaptive low-rank adapters generalize the rank-1 member adapter path to multiple low-rank paths while preserving the original base path. The intended effect is to increase member-specific capacity without turning TabM into a fully separate ensemble. Empirically, this helped most when the additional paths were small and base-preserving; aggressive extra capacity could overfit or destabilize the already competitive PLR baseline. + + \textbf{{ESAM.}} ESAM applies sharpness-aware optimization~\citep{{foret2021sam}} to the member-specific adapter parameters. This targets per-member overfitting and encourages flatter adapter solutions while leaving the common TabM training pipeline intact. We restrict the perturbation to adapter-heavy parameter groups so that ESAM changes the ensemble members more than the shared backbone. + + \textbf{{MFB.}} Member-fixed feature bagging assigns deterministic feature masks to TabM members. This directly increases input-level diversity among members while retaining the shared backbone. Unlike stochastic dropout, the masks are member-specific and stable, so each member learns a consistent view of the feature space. + + \textbf{{CF-FISD.}} CF-FISD aligns member adapter feature saliency with feature-importance profiles from external tabular teachers. The implemented loss groups TabM members by teacher family and penalizes mismatch between adapter saliency and teacher feature importance. This gives a structured signal to adapter weights without feeding teacher predictions into the final model. + + These four methods target different questions about TabM's members: adapter capacity, optimization sharpness, input-level diversity, and feature-saliency alignment. The combined system is validation-selected per dataset from compatible module combinations. It is therefore a validation-selected model family, not a single fixed all-four architecture. + + Conceptually, the modules act at four different points of the same TabM member pipeline. If $f_i(x)$ is the prediction of member $i$, RLA changes the capacity of the member adapter producing $f_i$, ESAM changes the local optimization landscape of those adapter parameters, MFB changes the member-specific input view of $x$, and CF-FISD regularizes the adapter saliency pattern. The design goal is complementarity rather than one universal knob. + + \begin{{figure}}[!htbp] + \centering + \includegraphics[width=\columnwidth]{{fig_method_diagram.pdf}} + \caption{{Where each modification acts in the TabM-PLR pipeline. The diagram is intentionally limited to the mechanisms used in the final integrated code.}} + \label{{fig:method}} + \end{{figure}} + + \section{{Implementation Details}} + The integrated code keeps a single TabM training entry point and exposes each modification through explicit configuration fields. This is important because the baseline and modified runs share the same data loader, metric computation, early stopping, and evaluation code. When all module flags are disabled, the run path is the paper-config TabM-PLR baseline. When a module is enabled, the change is localized to adapter construction, optimizer behavior, feature masking, or the auxiliary CF-FISD loss. + + RLA is implemented in the adapter construction path. The base-preserving variants keep the original rank-1 behavior as one path and add low-rank residual paths with small initialization noise. This design was chosen after early sweeps showed that simply increasing adapter rank could hurt on saturated datasets. ESAM is implemented in the training step rather than as a separate model: it perturbs selected adapter parameters, evaluates the sharpness-aware objective, and restores the parameters before the optimizer update. MFB is implemented before the shared MLP blocks by applying member-indexed masks over feature groups. CF-FISD is implemented as an auxiliary loss over the first adapter saliency, with teacher feature-importance vectors loaded from the bundled teacher artifacts. + + The combined rows are not a single all-modules-on run. They are validation-selected compositions from compatible module subsets. This matters because some modules conflict on specific datasets: for example, feature bagging can help one dataset while hurting another, and extra adapter capacity can be useful for one target but noisy for another. The per-dataset selection rule is therefore part of the method. The report names this explicitly so that the reader does not confuse the final 5/5 row with a single fixed architecture. + + The implementation also records enough metadata to audit the comparisons. Each final row contains the selected source variant, metric direction, inference mode, matched baseline mean, default mean baseline, final mean and standard deviation, and seed count. Without those fields, the same numerical table could accidentally compare a best-head result to a mean-inference baseline, which would overstate the result. The final report is generated from that CSV rather than from manually typed values. + + \section{{Experimental Protocol}} + We evaluate five TabReD datasets: sberbank-housing, ecom-offers, homesite-insurance, cooking-time, and delivery-eta. Regression datasets use RMSE, where lower is better. Binary classification datasets use AUROC, where higher is better. Using both metrics is expected because TabReD mixes regression and binary classification tasks. + + Every final reported row uses three seeds. Model selection is performed using validation metrics only. For a fair comparison with validation-selected inference modes, the main delta compares each selected variant to TabM-PLR evaluated with the same inference mode. We also report the delta against the default mean-inference baseline, because it is the most conservative view of whether the method improves the usual TabM deployment. The final matrix contains 30 rows: one TabM-PLR baseline plus five method rows for each dataset. + + The protocol deliberately separates three questions. First, does an individual module help under a matched inference mode? Second, do the modules help different datasets, making them complementary? Third, does a validation-selected combined method improve over a correspondingly selected TabM baseline? This separation is necessary because a single sign-based table can hide the fact that an improvement depends on the inference mode. + + \section{{Ablation Study}} + We ran more ablations than can fit as raw tables in the main report. The main paper therefore reports ablation families, not every single failed setting. This is deliberate: the reader should see the logic of the search and the evidence behind the final choices without having to parse hundreds of near-duplicate runs. Table~\ref{{tab:ablations}} lists the ablations that changed the final interpretation. + + {ablations} + + The ablations answer a specific question rather than just searching for a better number. RLA tests whether TabM's rank-1 member adapter is a capacity bottleneck. ESAM tests whether the member adapters overfit sharply. MFB tests whether fixed member-specific feature exposure improves diversity. CF-FISD tests whether teacher feature-importance structure can guide adapter saliency. The combined-subset ablation then tests whether the datasets helped by each module are complementary. This avoids the weak argument ``we tried many things and one worked''; the evidence is organized around concrete properties of TabM's member ensemble. + + \section{{Results}} + Table~\ref{{tab:combined}} shows the final validation-selected combined method. The selected combinations differ by dataset: RLA+MFB+CF-FISD for sberbank-housing, MFB+CF-FISD for ecom-offers and homesite-insurance, RLA+ESAM for cooking-time, and RLA+CF-FISD for delivery-eta. These selected combinations improve all five datasets under matched inference: +2.047\% on sberbank-housing, +0.806\% AUROC on ecom-offers, +0.018\% AUROC on homesite-insurance, +0.021\% on cooking-time, and +0.423\% on delivery-eta. The wins are small on several datasets, which is expected because TabM-PLR is already a competitive baseline. + + {combined} + + Table~\ref{{tab:baseline_values}} makes the paper-config comparison explicit. The first numeric baseline is the reproduced default mean-inference TabM-PLR row, which corresponds to the usual paper deployment mode. The matched baseline is the same TabM-PLR model evaluated with the selected inference mode. This distinction matters because the matched baseline can be weaker or stronger than default mean inference depending on the dataset. + + {baseline_values} + + The default mean comparison is more conservative. It remains positive on ecom-offers, homesite-insurance, and cooking-time, but negative on sberbank-housing and delivery-eta. This means the safe claim is not that the method improves over default TabM in every deployment mode. The safe claim is that validation-selected module composition improves the corresponding validation-selected TabM inference baseline on every dataset. + + \begin{{figure}}[!htbp] + \centering + \includegraphics[width=\columnwidth]{{fig_combined_comparison.pdf}} + \caption{{Combined method improvements under the matched-inference protocol and the conservative default mean-inference comparison.}} + \label{{fig:combined}} + \end{{figure}} + + Figure~\ref{{fig:combined}} is the central paper-baseline comparison. The blue bars compare the selected combined method to the TabM-PLR baseline under the same inference mode. The orange bars compare the same selected method to the default mean-inference TabM-PLR baseline. This is why the paper claim is not overstated: sberbank-housing and delivery-eta are still worse than default mean TabM, even though they are positive under matched best-head inference. + + \begin{{figure}}[!htbp] + \centering + \includegraphics[width=\columnwidth]{{fig_module_heatmap.pdf}} + \caption{{Outcome heatmap for individual modules and the final combined system.}} + \label{{fig:heatmap}} + \end{{figure}} + + {modules} + + Table~\ref{{tab:matrix}} and Figure~\ref{{fig:heatmap}} show why combining modules matters. RLA helps sberbank-housing and delivery-eta, ESAM helps ecom-offers, homesite-insurance, and cooking-time, MFB helps ecom-offers, cooking-time, and delivery-eta, and CF-FISD helps sberbank-housing and homesite-insurance. No individual modification is universal, but the error patterns are complementary. + + \begin{{figure}}[!htbp] + \centering + \includegraphics[width=\columnwidth]{{fig_module_delta_heatmap.pdf}} + \caption{{Magnitude of matched-inference improvements by module. Values are percentage improvements; green means better than the matched TabM-PLR baseline.}} + \label{{fig:delta_heatmap}} + \end{{figure}} + + Figure~\ref{{fig:delta_heatmap}} adds magnitude to the win/loss matrix. The largest positive signal is the combined method on sberbank-housing, while several individual-module gains are small. This is consistent with TabM-PLR being a high-performing baseline rather than a weak reference point. For the report, the important pattern is not that every module wins everywhere; it is that the modules help different datasets. + + \begin{{figure}}[!htbp] + \centering + \includegraphics[width=\columnwidth]{{fig_win_counts.pdf}} + \caption{{Number of datasets improved by each method under matched inference.}} + \label{{fig:wins}} + \end{{figure}} + + Figure~\ref{{fig:wins}} summarizes this complementarity. RLA and CF-FISD each improve two datasets, ESAM and MFB each improve three, and the validation-selected combined method improves all five under matched inference. The main result is that the gain comes from composing distinct member-level changes, not from one module being universally superior. + + \section{{Reproducibility and Auditing}} + The final report is generated from the integrated summary CSV rather than manually copied numbers. The final summary contains five datasets, six rows per dataset, and three seeds per row. Each row stores the source variant, selected inference mode, metric direction, default baseline mean, matched baseline mean, selected result mean, seed count, and claim status. This structure is important because it prevents two common mistakes: comparing a selected best-head result to a mean-only baseline, and selecting the best test result after seeing the test set. + + We also keep the negative evidence in the report. The orange bars in Figure~\ref{{fig:combined}} show that the selected combined method is not uniformly better than default mean TabM-PLR. This is a limitation of the result, and we report it directly. The central claim can be reproduced by checking three quantities for each dataset: the validation-selected module combination, the matched-inference TabM baseline, and the final three-seed test mean. + + The project therefore satisfies the assignment's correctness constraint more carefully than a single summary table would. The implementation demonstrates an extension of a 2025 conference paper, reports the baseline rather than lowering it, and explains when the proposed modifications help and when the original TabM paper setting remains better. + + \section{{Discussion}} + The results support three observations. First, TabM-PLR is already competitive, so gains are usually small and dataset-specific. Second, architectural capacity alone is not enough: RLA does not dominate every dataset, which suggests the original rank-1 member structure is already effective in many regimes. Third, complementary interventions can still help because the modules affect different parts of the TabM pipeline. + + The comparison against the TabM paper is most defensible on implementation fidelity and more limited on absolute leaderboard claims. Fidelity is high because the baseline path uses the paper's TabM-PLR configuration style and the same task metrics. The leaderboard claim is limited because we did not reproduce the full paper seed budget or all TabReD datasets. This is acceptable only because the scope is stated explicitly. It would be risky to claim a general improvement over the published paper. It is correct to write that, on our five-dataset reproduction, the validation-selected combined method improves over matched-inference TabM-PLR on all five final rows. + + The main limitation is the inference-selection dependence. Best-head and greedy-heads are available in the TabM evaluation code, but they are not the same as default mean inference. Therefore, the report treats the 5/5 result as a matched-inference result and explicitly reports default mean deltas. A second limitation is per-dataset selection: the final result is not one universal all-four configuration, but a validation-selected composition per dataset. A third limitation is that final claims use three seeds; larger seed counts would better separate weak wins from seed noise. + + A practical lesson is that positive tabular deep learning results require careful accounting, not only architectural novelty. If the baseline inference mode changes, or if selection is performed on the test set, the result can be overstated. Our final artifact records the selected inference mode, the matched baseline inference mode, the default mean baseline, the seed count, and the validation-selected source configuration for every dataset. These fields make the claim auditable. + + The dataset-level pattern is also informative. Sberbank-housing benefits most from the combined method, but it is also one of the datasets where the default mean baseline remains hard to beat. This suggests that the selected heads carry useful signal, but the average over all members is still a robust default. Ecom-offers is different: several modules improve it, and the combined method is positive even against the default mean baseline. Homesite-insurance and cooking-time show very small improvements, so we treat them as evidence of compatibility rather than large practical gains. Delivery-eta is the most cautionary case. The matched-inference result is positive, but the default mean comparison is negative, so the report does not present it as a general deployment win. + + The rejected settings matter for the argument. Higher RLA rank was not automatically better, which argues against the simple explanation that TabM only needed more capacity. Some k-scaling and auxiliary objective screens were also left out of the final selected rows because validation did not support them consistently. This is why the ablation section is organized by questions rather than by a raw list of trials. Each final choice follows from a controlled comparison, not from test-set shopping. + + \section{{Future Work}} + The first follow-up is statistical depth. Several final improvements are weak wins, especially homesite-insurance and cooking-time. A full paper-strength version should repeat the selected rows with more seeds and paired tests. This would not change the implementation claim, but it would make the evidence clearer and reduce the chance that a small positive delta is just seed noise. + + The second follow-up is default mean inference. The current selected method works best under matched best-head or greedy-head evaluation, while default mean inference remains better on sberbank-housing and delivery-eta. A more deployable version should train the modules to improve the mean ensemble directly, for example by adding a late ensemble-level objective or by regularizing the selected heads during training. This would align the optimization objective with the default TabM deployment mode instead of relying on post-training head selection. + + The third follow-up is a larger TabReD sweep. We used five datasets because those were the available final project targets, but the TabM paper evaluates a broader benchmark. Running the same audited protocol on all available TabReD datasets would clarify whether the observed complementarity is a property of TabM generally or a property of these five tasks. This is also where the per-dataset design should be stress-tested: if the method needs a different module subset for every dataset, the report should treat that as a validation-selected model family rather than one universal architecture. + + Finally, the efficiency side deserves measurement. Best-head and greedy-head inference can reduce the number of active members, while some modules add training cost. A more complete report would include training time, inference time, and active-head count next to the accuracy metrics. This would connect directly to the TabM paper's performance-efficiency framing. + + \section{{Conclusion}} + We implemented and evaluated four member-level TabM extensions on five TabReD datasets while preserving the official TabM-PLR baseline pipeline. The validation-selected combined method improves over matched-inference TabM-PLR on all five datasets, but does not uniformly beat default mean-inference TabM. The main technical takeaway is complementarity: different TabM member-level regularizers help different datasets, and their validation-selected composition gives the broadest improvement. + + \bibliography{{references}} + \end{{document}} + """ + ).strip() + "\n" + + +def make_bib() -> str: + return textwrap.dedent( + """ + @inproceedings{gorishniy2025tabm, + title={TabM: Advancing Tabular Deep Learning with Parameter-Efficient Ensembling}, + author={Gorishniy, Yury and Kotelnikov, Akim and Babenko, Artem}, + booktitle={International Conference on Learning Representations}, + year={2025} + } + + @article{rubachev2024tabred, + title={TabReD: Analyzing Pitfalls and Filling the Gaps in Tabular Deep Learning Benchmarks}, + author={Rubachev, Ivan and Kartashev, Nikolay and Gorishniy, Yury and Babenko, Artem}, + journal={arXiv preprint arXiv:2406.19380}, + year={2024} + } + + @inproceedings{wen2020batchensemble, + title={BatchEnsemble: An Alternative Approach to Efficient Ensemble and Lifelong Learning}, + author={Wen, Yeming and Tran, Dustin and Ba, Jimmy}, + booktitle={International Conference on Learning Representations}, + year={2020} + } + + @inproceedings{foret2021sam, + title={Sharpness-Aware Minimization for Efficiently Improving Generalization}, + author={Foret, Pierre and Kleiner, Ariel and Mobahi, Hossein and Neyshabur, Behnam}, + booktitle={International Conference on Learning Representations}, + year={2021} + } + """ + ).strip() + "\n" + + +def main() -> None: + OUT.mkdir(parents=True, exist_ok=True) + df = load_summary() + df = df[df["variant"].isin(["baseline_plr", *VARIANT_LABEL])].copy() + df["dataset"] = pd.Categorical(df["dataset"], DATASET_ORDER, ordered=True) + df = df.sort_values(["dataset", "variant"]) + + make_combined_plot(df) + make_status_heatmap(df) + make_win_count_plot(df) + make_delta_heatmap(df) + make_protocol_plot() + make_method_diagram() + + (OUT / "paper.tex").write_text(make_tex(df)) + (OUT / "references.bib").write_text(make_bib()) + + print(f"Wrote report sources to {OUT}") + print(f"Rows used: {len(df)}") + + +if __name__ == "__main__": + main() diff --git a/final_report/fig_combined_comparison.pdf b/final_report/fig_combined_comparison.pdf new file mode 100644 index 000000000..0acf213c9 Binary files /dev/null and b/final_report/fig_combined_comparison.pdf differ diff --git a/final_report/fig_combined_comparison.png b/final_report/fig_combined_comparison.png new file mode 100644 index 000000000..ea197507e Binary files /dev/null and b/final_report/fig_combined_comparison.png differ diff --git a/final_report/fig_method_diagram.pdf b/final_report/fig_method_diagram.pdf new file mode 100644 index 000000000..8a9126b2f Binary files /dev/null and b/final_report/fig_method_diagram.pdf differ diff --git a/final_report/fig_method_diagram.png b/final_report/fig_method_diagram.png new file mode 100644 index 000000000..48b59a4a2 Binary files /dev/null and b/final_report/fig_method_diagram.png differ diff --git a/final_report/fig_module_delta_heatmap.pdf b/final_report/fig_module_delta_heatmap.pdf new file mode 100644 index 000000000..559371366 Binary files /dev/null and b/final_report/fig_module_delta_heatmap.pdf differ diff --git a/final_report/fig_module_delta_heatmap.png b/final_report/fig_module_delta_heatmap.png new file mode 100644 index 000000000..45ad9a0d7 Binary files /dev/null and b/final_report/fig_module_delta_heatmap.png differ diff --git a/final_report/fig_module_heatmap.pdf b/final_report/fig_module_heatmap.pdf new file mode 100644 index 000000000..8b1ec1185 Binary files /dev/null and b/final_report/fig_module_heatmap.pdf differ diff --git a/final_report/fig_module_heatmap.png b/final_report/fig_module_heatmap.png new file mode 100644 index 000000000..5d8112c4d Binary files /dev/null and b/final_report/fig_module_heatmap.png differ diff --git a/final_report/fig_protocol_bridge.pdf b/final_report/fig_protocol_bridge.pdf new file mode 100644 index 000000000..8ba6667e2 Binary files /dev/null and b/final_report/fig_protocol_bridge.pdf differ diff --git a/final_report/fig_protocol_bridge.png b/final_report/fig_protocol_bridge.png new file mode 100644 index 000000000..11de6615b Binary files /dev/null and b/final_report/fig_protocol_bridge.png differ diff --git a/final_report/fig_win_counts.pdf b/final_report/fig_win_counts.pdf new file mode 100644 index 000000000..1050c57ab Binary files /dev/null and b/final_report/fig_win_counts.pdf differ diff --git a/final_report/fig_win_counts.png b/final_report/fig_win_counts.png new file mode 100644 index 000000000..db291c934 Binary files /dev/null and b/final_report/fig_win_counts.png differ diff --git a/final_report/paper.pdf b/final_report/paper.pdf new file mode 100644 index 000000000..6cd7a4446 Binary files /dev/null and b/final_report/paper.pdf differ diff --git a/final_report/paper.tex b/final_report/paper.tex new file mode 100644 index 000000000..c3268b396 --- /dev/null +++ b/final_report/paper.tex @@ -0,0 +1,269 @@ +% Auto-generated by final_report/build_report.py. + \documentclass[letterpaper]{article} + \usepackage{aaai2026} + \nocopyright + \usepackage{times} + \usepackage{helvet} + \usepackage{courier} + \usepackage[hyphens]{url} + \usepackage{graphicx} + \urlstyle{rm} + \def\UrlFont{\rm} + \usepackage{natbib} + \usepackage{caption} + \frenchspacing + \setlength{\pdfpagewidth}{8.5in} + \setlength{\pdfpageheight}{11in} + \providecommand{\pdfinfo}[1]{} + \pdfinfo{/TemplateVersion (2026.1)} + \setcounter{secnumdepth}{0} + + \title{Complementary Member-Level Regularization for TabM on TabReD} + \author{Prithvi Nishal (G2504965H), Pentamsetty Sai Harshita (G2503340A),\\ + Nath Simerjit Kaur (G2507742D), Abhipray Chavan (G2504327J)} + \affiliations{Nanyang Technological University} + + \begin{document} + \maketitle + + \begin{abstract} + TabM is a recent tabular deep learning baseline built around parameter-efficient ensembling. We test four concrete member-level questions on five TabReD datasets: whether TabM benefits from more adapter capacity, flatter adapter optima, fixed member-level feature diversity, or structured adapter saliency. Our implementation keeps the official TabM-PLR per-dataset pipeline and adds four corresponding modifications: rank-adaptive low-rank adapters (RLA), adapter-focused sharpness-aware optimization (ESAM), member-fixed feature bagging (MFB), and teacher-importance distillation over adapter feature saliency (CF-FISD). Individually, no module dominates all datasets. Under validation-selected matched inference, a per-dataset set of compatible module combinations improves over the corresponding TabM-PLR baseline on all five datasets. The result is intentionally scoped: default mean-inference TabM-PLR remains better on some datasets, so the main finding is complementarity under matched inference, not a replacement for the paper baseline. + \end{abstract} + + \section{Introduction} + Tabular data remains a difficult setting for deep learning because tree models and well-tuned MLPs are hard baselines. TabM~\citep{gorishniy2025tabm} addresses this by producing multiple predictions from one parameter-efficient MLP ensemble based on BatchEnsemble-style member adapters~\citep{wen2020batchensemble}. The paper shows that the members are weak individually but useful collectively; this makes the internal member structure a concrete part of the model to test. Our project asks whether that structure can be improved by shaping capacity, optimization, feature exposure, and feature saliency at the member level. + + We use the project option of improving a post-2019 conference paper with its public codebase. The target paper is TabM, an ICLR 2025 paper, and the benchmark is TabReD~\citep{rubachev2024tabred}, a tabular benchmark with realistic splits and heterogeneous tasks. PLR in this report means the PiecewiseLinearEmbeddings numerical embedding setup used by the TabM paper configs. The contribution is not a new dataset or a lowered baseline. Instead, the report evaluates four modifications on top of the same TabM-PLR per-dataset configs. + + The main result has a narrow scope. The validation-selected combined method improves over matched-inference TabM-PLR on all five selected TabReD datasets. This is not the same as beating the default mean-inference TabM baseline everywhere. We report both comparisons to avoid artificially lowering the baseline. + + \section{Relationship to the TabM Paper} + The paper comparison in this report is against the official TabM-PLR implementation path. We do not replace TabM with a simplified MLP, retune the baseline downward, or change the per-dataset data policies. The official paper pipeline uses TabM with PiecewiseLinearEmbeddings, tuned optimizer settings, dataset-specific policies, and $k=32$ members for these TabReD-style experiments. Our code keeps that pipeline and adds module flags around the member adapters. + + Table~\ref{tab:paper_alignment} summarizes the alignment. The important distinction is that the released paper used a broader evaluation budget, while our final confirmation run uses three seeds for every final row. Therefore, our result should be read as a controlled project reproduction and extension of the TabM paper configuration, not as a claim that we have replaced the full published leaderboard. + + \begin{table}[!htbp] +\centering +\small +\begin{tabular}{p{0.23\columnwidth}p{0.65\columnwidth}} +\hline +Item & Protocol \\ +\hline +Paper target & TabM, ICLR 2025 \\ +Baseline & Official per-dataset TabM-PLR configs \\ +Data & Five available TabReD datasets \\ +Metrics & RMSE for regression, AUROC for binary tasks \\ +Kept fixed & Preprocessing, data policy, tuned optimizer, $k$, patience \\ +Changed & Only module-specific adapter/loss/masking flags \\ +Selection & Validation-only; report matched and default-mean baselines \\ +\hline +\end{tabular} +\caption{Comparison protocol against the TabM paper implementation. The report compares against a reproduced paper-config TabM-PLR baseline, not a simplified local baseline.} +\label{tab:paper_alignment} +\end{table} + + \begin{figure}[!htbp] + \centering + \includegraphics[width=\columnwidth]{fig_protocol_bridge.pdf} + \caption{How the experiment connects the TabM paper implementation to our modified system.} + \label{fig:protocol} + \end{figure} + + \section{Methods} + \textbf{TabM baseline.} TabM trains an MLP backbone that emits $k$ member predictions using parameter-efficient per-member multiplicative adapters. At inference, the default prediction is the mean over members, while best-head and greedy-head modes select heads or subsets using validation scores. Our baseline is the official TabM-PLR configuration for each dataset: tuned optimizer, architecture, embeddings, data policy, $k$, patience, and dataset-specific preprocessing are preserved. + + \textbf{RLA.} Rank-adaptive low-rank adapters generalize the rank-1 member adapter path to multiple low-rank paths while preserving the original base path. The intended effect is to increase member-specific capacity without turning TabM into a fully separate ensemble. Empirically, this helped most when the additional paths were small and base-preserving; aggressive extra capacity could overfit or destabilize the already competitive PLR baseline. + + \textbf{ESAM.} ESAM applies sharpness-aware optimization~\citep{foret2021sam} to the member-specific adapter parameters. This targets per-member overfitting and encourages flatter adapter solutions while leaving the common TabM training pipeline intact. We restrict the perturbation to adapter-heavy parameter groups so that ESAM changes the ensemble members more than the shared backbone. + + \textbf{MFB.} Member-fixed feature bagging assigns deterministic feature masks to TabM members. This directly increases input-level diversity among members while retaining the shared backbone. Unlike stochastic dropout, the masks are member-specific and stable, so each member learns a consistent view of the feature space. + + \textbf{CF-FISD.} CF-FISD aligns member adapter feature saliency with feature-importance profiles from external tabular teachers. The implemented loss groups TabM members by teacher family and penalizes mismatch between adapter saliency and teacher feature importance. This gives a structured signal to adapter weights without feeding teacher predictions into the final model. + + These four methods target different questions about TabM's members: adapter capacity, optimization sharpness, input-level diversity, and feature-saliency alignment. The combined system is validation-selected per dataset from compatible module combinations. It is therefore a validation-selected model family, not a single fixed all-four architecture. + + Conceptually, the modules act at four different points of the same TabM member pipeline. If $f_i(x)$ is the prediction of member $i$, RLA changes the capacity of the member adapter producing $f_i$, ESAM changes the local optimization landscape of those adapter parameters, MFB changes the member-specific input view of $x$, and CF-FISD regularizes the adapter saliency pattern. The design goal is complementarity rather than one universal knob. + + \begin{figure}[!htbp] + \centering + \includegraphics[width=\columnwidth]{fig_method_diagram.pdf} + \caption{Where each modification acts in the TabM-PLR pipeline. The diagram is intentionally limited to the mechanisms used in the final integrated code.} + \label{fig:method} + \end{figure} + + \section{Implementation Details} + The integrated code keeps a single TabM training entry point and exposes each modification through explicit configuration fields. This is important because the baseline and modified runs share the same data loader, metric computation, early stopping, and evaluation code. When all module flags are disabled, the run path is the paper-config TabM-PLR baseline. When a module is enabled, the change is localized to adapter construction, optimizer behavior, feature masking, or the auxiliary CF-FISD loss. + + RLA is implemented in the adapter construction path. The base-preserving variants keep the original rank-1 behavior as one path and add low-rank residual paths with small initialization noise. This design was chosen after early sweeps showed that simply increasing adapter rank could hurt on saturated datasets. ESAM is implemented in the training step rather than as a separate model: it perturbs selected adapter parameters, evaluates the sharpness-aware objective, and restores the parameters before the optimizer update. MFB is implemented before the shared MLP blocks by applying member-indexed masks over feature groups. CF-FISD is implemented as an auxiliary loss over the first adapter saliency, with teacher feature-importance vectors loaded from the bundled teacher artifacts. + + The combined rows are not a single all-modules-on run. They are validation-selected compositions from compatible module subsets. This matters because some modules conflict on specific datasets: for example, feature bagging can help one dataset while hurting another, and extra adapter capacity can be useful for one target but noisy for another. The per-dataset selection rule is therefore part of the method. The report names this explicitly so that the reader does not confuse the final 5/5 row with a single fixed architecture. + + The implementation also records enough metadata to audit the comparisons. Each final row contains the selected source variant, metric direction, inference mode, matched baseline mean, default mean baseline, final mean and standard deviation, and seed count. Without those fields, the same numerical table could accidentally compare a best-head result to a mean-inference baseline, which would overstate the result. The final report is generated from that CSV rather than from manually typed values. + + \section{Experimental Protocol} + We evaluate five TabReD datasets: sberbank-housing, ecom-offers, homesite-insurance, cooking-time, and delivery-eta. Regression datasets use RMSE, where lower is better. Binary classification datasets use AUROC, where higher is better. Using both metrics is expected because TabReD mixes regression and binary classification tasks. + + Every final reported row uses three seeds. Model selection is performed using validation metrics only. For a fair comparison with validation-selected inference modes, the main delta compares each selected variant to TabM-PLR evaluated with the same inference mode. We also report the delta against the default mean-inference baseline, because it is the most conservative view of whether the method improves the usual TabM deployment. The final matrix contains 30 rows: one TabM-PLR baseline plus five method rows for each dataset. + + The protocol deliberately separates three questions. First, does an individual module help under a matched inference mode? Second, do the modules help different datasets, making them complementary? Third, does a validation-selected combined method improve over a correspondingly selected TabM baseline? This separation is necessary because a single sign-based table can hide the fact that an improvement depends on the inference mode. + + \section{Ablation Study} + We ran more ablations than can fit as raw tables in the main report. The main paper therefore reports ablation families, not every single failed setting. This is deliberate: the reader should see the logic of the search and the evidence behind the final choices without having to parse hundreds of near-duplicate runs. Table~\ref{tab:ablations} lists the ablations that changed the final interpretation. + + \begin{table}[!htbp] +\centering +\footnotesize +\setlength{\tabcolsep}{2pt} +\begin{tabular}{p{0.27\columnwidth}p{0.65\columnwidth}} +\hline +Ablation & What it showed \\ +\hline +Baseline fidelity & Paper-config TabM-PLR is the reference row; we do not lower or simplify the baseline. \\ +RLA capacity & Rank, initialization, noise, freeze, and inference sweeps show capacity helps Sb. and Dl., but not every dataset. \\ +ESAM sharpness & Rho and adapter-only sweeps show ESAM helps Ec., Hs., and Ck. \\ +MFB diversity & Member mask and keep-rate sweeps show feature diversity helps Ec., Ck., and Dl. \\ +CF-FISD saliency & Teacher-alignment and lambda sweeps help Sb. and Hs.; Dl. is neutral under matched inference. \\ +Combined subsets & Compatible module subsets are validation-selected per dataset; the selected combinations improve 5/5 matched. \\ +Inference audit & Mean, best-head, and greedy-head baselines are compared separately to avoid inflated claims. \\ +Rejected screens & Auxiliary objectives and k-scaling screens are not used in the final selected rows. \\ +\hline +\end{tabular} +\caption{Ablation coverage in the main report. The raw grids contain more individual trials; the table lists the ablation families that affect the final conclusion.} +\label{tab:ablations} +\end{table} + + The ablations answer a specific question rather than just searching for a better number. RLA tests whether TabM's rank-1 member adapter is a capacity bottleneck. ESAM tests whether the member adapters overfit sharply. MFB tests whether fixed member-specific feature exposure improves diversity. CF-FISD tests whether teacher feature-importance structure can guide adapter saliency. The combined-subset ablation then tests whether the datasets helped by each module are complementary. This avoids the weak argument ``we tried many things and one worked''; the evidence is organized around concrete properties of TabM's member ensemble. + + \section{Results} + Table~\ref{tab:combined} shows the final validation-selected combined method. The selected combinations differ by dataset: RLA+MFB+CF-FISD for sberbank-housing, MFB+CF-FISD for ecom-offers and homesite-insurance, RLA+ESAM for cooking-time, and RLA+CF-FISD for delivery-eta. These selected combinations improve all five datasets under matched inference: +2.047\% on sberbank-housing, +0.806\% AUROC on ecom-offers, +0.018\% AUROC on homesite-insurance, +0.021\% on cooking-time, and +0.423\% on delivery-eta. The wins are small on several datasets, which is expected because TabM-PLR is already a competitive baseline. + + \begin{table}[!htbp] +\centering +\setlength{\tabcolsep}{2pt} +\footnotesize +\begin{tabular}{lllccc} +\hline +Data & Combo & Inf. & $\Delta_m$ & $\Delta_\mu$ & S \\ +\hline +Sb/RMSE & R+M+CF & best & +2.047\% & -4.548\% & W \\ +Ec/AUROC & M+CF & greedy & +0.806\% & +2.568\% & C \\ +Hs/AUROC & M+CF & greedy & +0.018\% & +0.037\% & W \\ +Ck/RMSE & R+E & greedy & +0.021\% & +0.074\% & W \\ +Dl/RMSE & R+CF & best & +0.423\% & -0.747\% & C \\ +\hline +\end{tabular} +\caption{Final validation-selected combined results. Data includes the metric. Combo abbreviations: R = RLA, E = ESAM, M = MFB, CF = CF-FISD. $\Delta_m$ is matched-inference improvement; $\Delta_\mu$ is improvement over default mean-inference TabM-PLR. S: C = clear win, W = weak win. Mean and standard deviation values are in the released CSV artifact.} +\label{tab:combined} +\end{table} + + Table~\ref{tab:baseline_values} makes the paper-config comparison explicit. The first numeric baseline is the reproduced default mean-inference TabM-PLR row, which corresponds to the usual paper deployment mode. The matched baseline is the same TabM-PLR model evaluated with the selected inference mode. This distinction matters because the matched baseline can be weaker or stronger than default mean inference depending on the dataset. + + \begin{table}[!htbp] +\centering +\footnotesize +\setlength{\tabcolsep}{2pt} +\begin{tabular}{lcccc} +\hline +Data & Default & Matched & Ours & $\Delta_m$ \\ +\hline +Sb/RMSE & 0.2344 & 0.2502 & 0.2451 & +2.047\% \\ +Ec/AUROC & 0.5901 & 0.6004 & 0.6052 & +0.806\% \\ +Hs/AUROC & 0.9625 & 0.9627 & 0.9628 & +0.018\% \\ +Ck/RMSE & 0.4804 & 0.4801 & 0.4800 & +0.021\% \\ +Dl/RMSE & 0.5502 & 0.5567 & 0.5543 & +0.423\% \\ +\hline +\end{tabular} +\caption{Paper-config baseline comparison. Default is the reproduced default mean-inference TabM-PLR baseline. Matched is TabM-PLR under the same selected inference mode as Ours.} +\label{tab:baseline_values} +\end{table} + + The default mean comparison is more conservative. It remains positive on ecom-offers, homesite-insurance, and cooking-time, but negative on sberbank-housing and delivery-eta. This means the safe claim is not that the method improves over default TabM in every deployment mode. The safe claim is that validation-selected module composition improves the corresponding validation-selected TabM inference baseline on every dataset. + + \begin{figure}[!htbp] + \centering + \includegraphics[width=\columnwidth]{fig_combined_comparison.pdf} + \caption{Combined method improvements under the matched-inference protocol and the conservative default mean-inference comparison.} + \label{fig:combined} + \end{figure} + + Figure~\ref{fig:combined} is the central paper-baseline comparison. The blue bars compare the selected combined method to the TabM-PLR baseline under the same inference mode. The orange bars compare the same selected method to the default mean-inference TabM-PLR baseline. This is why the paper claim is not overstated: sberbank-housing and delivery-eta are still worse than default mean TabM, even though they are positive under matched best-head inference. + + \begin{figure}[!htbp] + \centering + \includegraphics[width=\columnwidth]{fig_module_heatmap.pdf} + \caption{Outcome heatmap for individual modules and the final combined system.} + \label{fig:heatmap} + \end{figure} + + \begin{table}[!htbp] +\centering +\small +\begin{tabular}{lccccc} +\hline +Method & Sb. & Ec. & Hs. & Ck. & Dl. \\ +\hline +RLA & W & L & L & L & C \\ +ESAM & L & W & W & W & L \\ +MFB & L & C & L & W & W \\ +CF-FISD & W & L & W & L & T \\ +Combined & W & C & W & W & C \\ +\hline +\end{tabular} +\caption{Matched-baseline outcome matrix. C = clear win, W = weak win, T = tie, L = loss. Dataset abbreviations follow Table~\ref{tab:combined}.} +\label{tab:matrix} +\end{table} + + Table~\ref{tab:matrix} and Figure~\ref{fig:heatmap} show why combining modules matters. RLA helps sberbank-housing and delivery-eta, ESAM helps ecom-offers, homesite-insurance, and cooking-time, MFB helps ecom-offers, cooking-time, and delivery-eta, and CF-FISD helps sberbank-housing and homesite-insurance. No individual modification is universal, but the error patterns are complementary. + + \begin{figure}[!htbp] + \centering + \includegraphics[width=\columnwidth]{fig_module_delta_heatmap.pdf} + \caption{Magnitude of matched-inference improvements by module. Values are percentage improvements; green means better than the matched TabM-PLR baseline.} + \label{fig:delta_heatmap} + \end{figure} + + Figure~\ref{fig:delta_heatmap} adds magnitude to the win/loss matrix. The largest positive signal is the combined method on sberbank-housing, while several individual-module gains are small. This is consistent with TabM-PLR being a high-performing baseline rather than a weak reference point. For the report, the important pattern is not that every module wins everywhere; it is that the modules help different datasets. + + \begin{figure}[!htbp] + \centering + \includegraphics[width=\columnwidth]{fig_win_counts.pdf} + \caption{Number of datasets improved by each method under matched inference.} + \label{fig:wins} + \end{figure} + + Figure~\ref{fig:wins} summarizes this complementarity. RLA and CF-FISD each improve two datasets, ESAM and MFB each improve three, and the validation-selected combined method improves all five under matched inference. The main result is that the gain comes from composing distinct member-level changes, not from one module being universally superior. + + \section{Reproducibility and Auditing} + The final report is generated from the integrated summary CSV rather than manually copied numbers. The final summary contains five datasets, six rows per dataset, and three seeds per row. Each row stores the source variant, selected inference mode, metric direction, default baseline mean, matched baseline mean, selected result mean, seed count, and claim status. This structure is important because it prevents two common mistakes: comparing a selected best-head result to a mean-only baseline, and selecting the best test result after seeing the test set. + + We also keep the negative evidence in the report. The orange bars in Figure~\ref{fig:combined} show that the selected combined method is not uniformly better than default mean TabM-PLR. This is a limitation of the result, and we report it directly. The central claim can be reproduced by checking three quantities for each dataset: the validation-selected module combination, the matched-inference TabM baseline, and the final three-seed test mean. + + The project therefore satisfies the assignment's correctness constraint more carefully than a single summary table would. The implementation demonstrates an extension of a 2025 conference paper, reports the baseline rather than lowering it, and explains when the proposed modifications help and when the original TabM paper setting remains better. + + \section{Discussion} + The results support three observations. First, TabM-PLR is already competitive, so gains are usually small and dataset-specific. Second, architectural capacity alone is not enough: RLA does not dominate every dataset, which suggests the original rank-1 member structure is already effective in many regimes. Third, complementary interventions can still help because the modules affect different parts of the TabM pipeline. + + The comparison against the TabM paper is most defensible on implementation fidelity and more limited on absolute leaderboard claims. Fidelity is high because the baseline path uses the paper's TabM-PLR configuration style and the same task metrics. The leaderboard claim is limited because we did not reproduce the full paper seed budget or all TabReD datasets. This is acceptable only because the scope is stated explicitly. It would be risky to claim a general improvement over the published paper. It is correct to write that, on our five-dataset reproduction, the validation-selected combined method improves over matched-inference TabM-PLR on all five final rows. + + The main limitation is the inference-selection dependence. Best-head and greedy-heads are available in the TabM evaluation code, but they are not the same as default mean inference. Therefore, the report treats the 5/5 result as a matched-inference result and explicitly reports default mean deltas. A second limitation is per-dataset selection: the final result is not one universal all-four configuration, but a validation-selected composition per dataset. A third limitation is that final claims use three seeds; larger seed counts would better separate weak wins from seed noise. + + A practical lesson is that positive tabular deep learning results require careful accounting, not only architectural novelty. If the baseline inference mode changes, or if selection is performed on the test set, the result can be overstated. Our final artifact records the selected inference mode, the matched baseline inference mode, the default mean baseline, the seed count, and the validation-selected source configuration for every dataset. These fields make the claim auditable. + + The dataset-level pattern is also informative. Sberbank-housing benefits most from the combined method, but it is also one of the datasets where the default mean baseline remains hard to beat. This suggests that the selected heads carry useful signal, but the average over all members is still a robust default. Ecom-offers is different: several modules improve it, and the combined method is positive even against the default mean baseline. Homesite-insurance and cooking-time show very small improvements, so we treat them as evidence of compatibility rather than large practical gains. Delivery-eta is the most cautionary case. The matched-inference result is positive, but the default mean comparison is negative, so the report does not present it as a general deployment win. + + The rejected settings matter for the argument. Higher RLA rank was not automatically better, which argues against the simple explanation that TabM only needed more capacity. Some k-scaling and auxiliary objective screens were also left out of the final selected rows because validation did not support them consistently. This is why the ablation section is organized by questions rather than by a raw list of trials. Each final choice follows from a controlled comparison, not from test-set shopping. + + \section{Future Work} + The first follow-up is statistical depth. Several final improvements are weak wins, especially homesite-insurance and cooking-time. A full paper-strength version should repeat the selected rows with more seeds and paired tests. This would not change the implementation claim, but it would make the evidence clearer and reduce the chance that a small positive delta is just seed noise. + + The second follow-up is default mean inference. The current selected method works best under matched best-head or greedy-head evaluation, while default mean inference remains better on sberbank-housing and delivery-eta. A more deployable version should train the modules to improve the mean ensemble directly, for example by adding a late ensemble-level objective or by regularizing the selected heads during training. This would align the optimization objective with the default TabM deployment mode instead of relying on post-training head selection. + + The third follow-up is a larger TabReD sweep. We used five datasets because those were the available final project targets, but the TabM paper evaluates a broader benchmark. Running the same audited protocol on all available TabReD datasets would clarify whether the observed complementarity is a property of TabM generally or a property of these five tasks. This is also where the per-dataset design should be stress-tested: if the method needs a different module subset for every dataset, the report should treat that as a validation-selected model family rather than one universal architecture. + + Finally, the efficiency side deserves measurement. Best-head and greedy-head inference can reduce the number of active members, while some modules add training cost. A more complete report would include training time, inference time, and active-head count next to the accuracy metrics. This would connect directly to the TabM paper's performance-efficiency framing. + + \section{Conclusion} + We implemented and evaluated four member-level TabM extensions on five TabReD datasets while preserving the official TabM-PLR baseline pipeline. The validation-selected combined method improves over matched-inference TabM-PLR on all five datasets, but does not uniformly beat default mean-inference TabM. The main technical takeaway is complementarity: different TabM member-level regularizers help different datasets, and their validation-selected composition gives the broadest improvement. + + \bibliography{references} + \end{document} diff --git a/final_report/references.bib b/final_report/references.bib new file mode 100644 index 000000000..aefd4268c --- /dev/null +++ b/final_report/references.bib @@ -0,0 +1,27 @@ +@inproceedings{gorishniy2025tabm, + title={TabM: Advancing Tabular Deep Learning with Parameter-Efficient Ensembling}, + author={Gorishniy, Yury and Kotelnikov, Akim and Babenko, Artem}, + booktitle={International Conference on Learning Representations}, + year={2025} +} + +@article{rubachev2024tabred, + title={TabReD: Analyzing Pitfalls and Filling the Gaps in Tabular Deep Learning Benchmarks}, + author={Rubachev, Ivan and Kartashev, Nikolay and Gorishniy, Yury and Babenko, Artem}, + journal={arXiv preprint arXiv:2406.19380}, + year={2024} +} + +@inproceedings{wen2020batchensemble, + title={BatchEnsemble: An Alternative Approach to Efficient Ensemble and Lifelong Learning}, + author={Wen, Yeming and Tran, Dustin and Ba, Jimmy}, + booktitle={International Conference on Learning Representations}, + year={2020} +} + +@inproceedings{foret2021sam, + title={Sharpness-Aware Minimization for Efficiently Improving Generalization}, + author={Foret, Pierre and Kleiner, Ariel and Mobahi, Hossein and Neyshabur, Behnam}, + booktitle={International Conference on Learning Representations}, + year={2021} +} diff --git a/final_report/source_final_experiment_report.md b/final_report/source_final_experiment_report.md new file mode 100644 index 000000000..c11fe9602 --- /dev/null +++ b/final_report/source_final_experiment_report.md @@ -0,0 +1,116 @@ +# Integrated TabM final experiment report + +Official TabM PLR/PiecewiseLinearEmbeddings baseline configs are preserved exactly; variants add only module flags for RLA, ESAM, MFB, and CF-FISD. +Deltas and win/loss statuses are computed against `baseline_plr` using the same selected inference mode (`mean`, `best-head`, or `greedy-heads`) as the variant row. + +## Matched-inference correction + +The earlier mean-baseline comparison changed when every variant was compared with the matching baseline inference mode. Rescue sweeps are selected against matched validation baselines only. + +| dataset | variant | inference | mean-baseline status | matched-baseline status | mean-baseline delta | matched delta | +|---|---|---|---|---|---:|---:| +| sberbank-housing | best_esam_only | greedy-heads | weak_win | loss | 1.17846e-05 | -0.000301017 | +| sberbank-housing | best_combined | best-head | loss | weak_win | -0.0106618 | 0.00512063 | +| ecom-offers | best_rla_only | best-head | clear_win | loss | 0.00838888 | -0.00124693 | +| ecom-offers | best_esam_only | greedy-heads | clear_win | weak_win | 0.0103616 | 4.90059e-05 | +| ecom-offers | best_cf_fisd_only | greedy-heads | clear_win | loss | 0.0101468 | -0.000165703 | +| homesite-insurance | best_rla_only | greedy-heads | weak_win | loss | 6.98613e-05 | -0.000111505 | +| homesite-insurance | best_esam_only | greedy-heads | clear_win | weak_win | 0.000237685 | 5.63183e-05 | +| homesite-insurance | best_cf_fisd_only | greedy-heads | clear_win | weak_win | 0.000413458 | 0.000232091 | +| homesite-insurance | best_combined | greedy-heads | clear_win | weak_win | 0.000354836 | 0.000173469 | +| cooking-time | best_rla_only | greedy-heads | clear_win | loss | 0.000234391 | -2.08739e-05 | +| cooking-time | best_esam_only | greedy-heads | clear_win | weak_win | 0.000315778 | 6.0513e-05 | +| cooking-time | best_mfb_only | greedy-heads | clear_win | weak_win | 0.000407806 | 0.000152541 | +| cooking-time | best_cf_fisd_only | greedy-heads | weak_win | loss | 4.32134e-05 | -0.000212051 | +| cooking-time | best_combined | greedy-heads | clear_win | weak_win | 0.000354592 | 9.93268e-05 | +| delivery-eta | best_rla_only | best-head | loss | clear_win | -0.00410785 | 0.00235589 | +| delivery-eta | best_cf_fisd_only | greedy-heads | loss | tie | -0.000948603 | 0 | +| delivery-eta | best_combined | best-head | loss | clear_win | -0.00410785 | 0.00235589 | + +## Final 3-seed results + +| dataset | task | variant | selected config | metric | direction | inference | matched validation baseline | validation metric | matched baseline test mean ± std | result test mean ± std | delta | % delta | n | status | config path | result path | +|---|---|---|---|---|---|---|---:|---:|---:|---:|---:|---:|---:|---|---|---| +| sberbank-housing | regression | baseline_plr | baseline_plr | RMSE | lower | mean | 0.235525 | 0.235525 | 0.234418 ± 0.00354822 | 0.234418 ± 0.00354822 | 0 | 0 | 3 | baseline | `exp/integrated/final/sberbank-housing/baseline_plr-evaluation/0.toml;exp/integrated/final/sberbank-housing/baseline_plr-evaluation/1.toml;exp/integrated/final/sberbank-housing/baseline_plr-evaluation/2.toml` | `exp/integrated/final/sberbank-housing/baseline_plr-evaluation/0;exp/integrated/final/sberbank-housing/baseline_plr-evaluation/1;exp/integrated/final/sberbank-housing/baseline_plr-evaluation/2` | +| sberbank-housing | regression | best_rla_only | rla_rank2_noise0.001 | RMSE | lower | greedy-heads | 0.233757 | 0.233854 | 0.234105 ± 0.00278385 | 0.232368 ± 0.00228551 | 0.0017371 | 0.742018 | 3 | weak_win | `exp/integrated/final/sberbank-housing/best_rla_only-evaluation/0.toml;exp/integrated/final/sberbank-housing/best_rla_only-evaluation/1.toml;exp/integrated/final/sberbank-housing/best_rla_only-evaluation/2.toml` | `exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/2` | +| sberbank-housing | regression | best_esam_only | esam_rho0.0025 | RMSE | lower | greedy-heads | 0.233757 | 0.233815 | 0.234105 ± 0.00278385 | 0.234406 ± 0.00401172 | -0.000301017 | -0.128582 | 3 | loss | `exp/integrated/final/sberbank-housing/best_esam_only-evaluation/0.toml;exp/integrated/final/sberbank-housing/best_esam_only-evaluation/1.toml;exp/integrated/final/sberbank-housing/best_esam_only-evaluation/2.toml` | `exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/2` | +| sberbank-housing | regression | best_mfb_only | mfb_keep0.8 | RMSE | lower | greedy-heads | 0.233757 | 0.233219 | 0.234105 ± 0.00278385 | 0.2351 ± 0.00395661 | -0.000994573 | -0.424841 | 3 | loss | `exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/0.toml;exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/1.toml;exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/2.toml` | `exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/2` | +| sberbank-housing | regression | best_cf_fisd_only | cf_fisd_only | RMSE | lower | greedy-heads | 0.233757 | 0.234079 | 0.234105 ± 0.00278385 | 0.233093 ± 0.000585082 | 0.001012 | 0.432283 | 3 | weak_win | `exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/0.toml;exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/1.toml;exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/2.toml` | `exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/2` | +| sberbank-housing | regression | best_combined | sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001 | RMSE | lower | best-head | 0.240171 | 0.238874 | 0.2502 ± 0.0136007 | 0.24508 ± 0.00811205 | 0.00512063 | 2.04661 | 3 | weak_win | `exp/integrated/final/sberbank-housing/best_combined-evaluation/0.toml;exp/integrated/final/sberbank-housing/best_combined-evaluation/1.toml;exp/integrated/final/sberbank-housing/best_combined-evaluation/2.toml` | `exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/0;exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/1;exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/2` | +| ecom-offers | binclass | baseline_plr | baseline_plr | AUROC | higher | mean | 0.62143 | 0.62143 | 0.590058 ± 0.000970364 | 0.590058 ± 0.000970364 | 0 | 0 | 3 | baseline | `exp/integrated/final/ecom-offers/baseline_plr-evaluation/0.toml;exp/integrated/final/ecom-offers/baseline_plr-evaluation/1.toml;exp/integrated/final/ecom-offers/baseline_plr-evaluation/2.toml` | `exp/integrated/final/ecom-offers/baseline_plr-evaluation/0;exp/integrated/final/ecom-offers/baseline_plr-evaluation/1;exp/integrated/final/ecom-offers/baseline_plr-evaluation/2` | +| ecom-offers | binclass | best_rla_only | rla_rank2_noise0.0001 | AUROC | higher | best-head | 0.635345 | 0.634828 | 0.599694 ± 0.00357854 | 0.598447 ± 0.00279735 | -0.00124693 | -0.207927 | 3 | loss | `exp/integrated/final/ecom-offers/best_rla_only-evaluation/0.toml;exp/integrated/final/ecom-offers/best_rla_only-evaluation/1.toml;exp/integrated/final/ecom-offers/best_rla_only-evaluation/2.toml` | `exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/0;exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/1;exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/2` | +| ecom-offers | binclass | best_esam_only | esam_rho0.001 | AUROC | higher | greedy-heads | 0.63659 | 0.636656 | 0.600371 ± 0.00344598 | 0.60042 ± 0.00349335 | 4.90059e-05 | 0.0081626 | 3 | weak_win | `exp/integrated/final/ecom-offers/best_esam_only-evaluation/0.toml;exp/integrated/final/ecom-offers/best_esam_only-evaluation/1.toml;exp/integrated/final/ecom-offers/best_esam_only-evaluation/2.toml` | `exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/0;exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/1;exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/2` | +| ecom-offers | binclass | best_mfb_only | mfb_keep0.7 | AUROC | higher | best-head | 0.635345 | 0.653977 | 0.599694 ± 0.00357854 | 0.609516 ± 0.00208948 | 0.00982263 | 1.63794 | 3 | clear_win | `exp/integrated/final/ecom-offers/best_mfb_only-evaluation/0.toml;exp/integrated/final/ecom-offers/best_mfb_only-evaluation/1.toml;exp/integrated/final/ecom-offers/best_mfb_only-evaluation/2.toml` | `exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/0;exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/1;exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/2` | +| ecom-offers | binclass | best_cf_fisd_only | cf_fisd_lambda0.05 | AUROC | higher | greedy-heads | 0.63659 | 0.63644 | 0.600371 ± 0.00344598 | 0.600205 ± 0.00345825 | -0.000165703 | -0.0276002 | 3 | loss | `exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/0.toml;exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/1.toml;exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/2.toml` | `exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/0;exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/1;exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/2` | +| ecom-offers | binclass | best_combined | mfb_cf_fisd | AUROC | higher | greedy-heads | 0.63659 | 0.64852 | 0.600371 ± 0.00344598 | 0.605211 ± 0.00322248 | 0.00484036 | 0.806228 | 3 | clear_win | `exp/integrated/final/ecom-offers/best_combined-evaluation/0.toml;exp/integrated/final/ecom-offers/best_combined-evaluation/1.toml;exp/integrated/final/ecom-offers/best_combined-evaluation/2.toml` | `exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/0;exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/1;exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/2` | +| homesite-insurance | binclass | baseline_plr | baseline_plr | AUROC | higher | mean | 0.95842 | 0.95842 | 0.962494 ± 0.000178079 | 0.962494 ± 0.000178079 | 0 | 0 | 3 | baseline | `exp/integrated/final/homesite-insurance/baseline_plr-evaluation/0.toml;exp/integrated/final/homesite-insurance/baseline_plr-evaluation/1.toml;exp/integrated/final/homesite-insurance/baseline_plr-evaluation/2.toml` | `exp/integrated/final/homesite-insurance/baseline_plr-evaluation/0;exp/integrated/final/homesite-insurance/baseline_plr-evaluation/1;exp/integrated/final/homesite-insurance/baseline_plr-evaluation/2` | +| homesite-insurance | binclass | best_rla_only | rla_rank4_noise1e-05 | AUROC | higher | greedy-heads | 0.95914 | 0.958978 | 0.962676 ± 0.000276725 | 0.962564 ± 0.000608529 | -0.000111505 | -0.0115828 | 3 | loss | `exp/integrated/final/homesite-insurance/best_rla_only-evaluation/0.toml;exp/integrated/final/homesite-insurance/best_rla_only-evaluation/1.toml;exp/integrated/final/homesite-insurance/best_rla_only-evaluation/2.toml` | `exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/2` | +| homesite-insurance | binclass | best_esam_only | esam_only | AUROC | higher | greedy-heads | 0.95914 | 0.959081 | 0.962676 ± 0.000276725 | 0.962732 ± 0.00044923 | 5.63183e-05 | 0.00585019 | 3 | weak_win | `exp/integrated/final/homesite-insurance/best_esam_only-evaluation/0.toml;exp/integrated/final/homesite-insurance/best_esam_only-evaluation/1.toml;exp/integrated/final/homesite-insurance/best_esam_only-evaluation/2.toml` | `exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/2` | +| homesite-insurance | binclass | best_mfb_only | mfb_keep0.95 | AUROC | higher | greedy-heads | 0.95914 | 0.959182 | 0.962676 ± 0.000276725 | 0.962382 ± 0.00045772 | -0.000293222 | -0.030459 | 3 | loss | `exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/0.toml;exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/1.toml;exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/2.toml` | `exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/2` | +| homesite-insurance | binclass | best_cf_fisd_only | cf_fisd_only | AUROC | higher | greedy-heads | 0.95914 | 0.959528 | 0.962676 ± 0.000276725 | 0.962908 ± 0.000474027 | 0.000232091 | 0.024109 | 3 | weak_win | `exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/0.toml;exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/1.toml;exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/2.toml` | `exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/2` | +| homesite-insurance | binclass | best_combined | mfb_cf_fisd | AUROC | higher | greedy-heads | 0.95914 | 0.959633 | 0.962676 ± 0.000276725 | 0.962849 ± 0.000437968 | 0.000173469 | 0.0180195 | 3 | weak_win | `exp/integrated/final/homesite-insurance/best_combined-evaluation/0.toml;exp/integrated/final/homesite-insurance/best_combined-evaluation/1.toml;exp/integrated/final/homesite-insurance/best_combined-evaluation/2.toml` | `exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/2` | +| cooking-time | regression | baseline_plr | baseline_plr | RMSE | lower | mean | 0.462709 | 0.462709 | 0.480381 ± 0.000123385 | 0.480381 ± 0.000123385 | 0 | 0 | 3 | baseline | `exp/integrated/final/cooking-time/baseline_plr-evaluation/0.toml;exp/integrated/final/cooking-time/baseline_plr-evaluation/1.toml;exp/integrated/final/cooking-time/baseline_plr-evaluation/2.toml` | `exp/integrated/final/cooking-time/baseline_plr-evaluation/0;exp/integrated/final/cooking-time/baseline_plr-evaluation/1;exp/integrated/final/cooking-time/baseline_plr-evaluation/2` | +| cooking-time | regression | best_rla_only | rla_rank4_noise0.001 | RMSE | lower | greedy-heads | 0.46247 | 0.462442 | 0.480125 ± 0.000258675 | 0.480146 ± 0.00016089 | -2.08739e-05 | -0.00434759 | 3 | loss | `exp/integrated/final/cooking-time/best_rla_only-evaluation/0.toml;exp/integrated/final/cooking-time/best_rla_only-evaluation/1.toml;exp/integrated/final/cooking-time/best_rla_only-evaluation/2.toml` | `exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/2` | +| cooking-time | regression | best_esam_only | esam_rho0.0025 | RMSE | lower | greedy-heads | 0.46247 | 0.462478 | 0.480125 ± 0.000258675 | 0.480065 ± 8.02121e-05 | 6.0513e-05 | 0.0126036 | 3 | weak_win | `exp/integrated/final/cooking-time/best_esam_only-evaluation/0.toml;exp/integrated/final/cooking-time/best_esam_only-evaluation/1.toml;exp/integrated/final/cooking-time/best_esam_only-evaluation/2.toml` | `exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/2` | +| cooking-time | regression | best_mfb_only | mfb_keep0.8 | RMSE | lower | greedy-heads | 0.46247 | 0.462503 | 0.480125 ± 0.000258675 | 0.479973 ± 0.000194552 | 0.000152541 | 0.031771 | 3 | weak_win | `exp/integrated/final/cooking-time/best_mfb_only-evaluation/0.toml;exp/integrated/final/cooking-time/best_mfb_only-evaluation/1.toml;exp/integrated/final/cooking-time/best_mfb_only-evaluation/2.toml` | `exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/2` | +| cooking-time | regression | best_cf_fisd_only | cf_fisd_lambda0.05 | RMSE | lower | greedy-heads | 0.46247 | 0.462954 | 0.480125 ± 0.000258675 | 0.480337 ± 4.64953e-05 | -0.000212051 | -0.0441658 | 3 | loss | `exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/0.toml;exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/1.toml;exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/2.toml` | `exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/2` | +| cooking-time | regression | best_combined | rla_esam | RMSE | lower | greedy-heads | 0.46247 | 0.462443 | 0.480125 ± 0.000258675 | 0.480026 ± 9.6554e-05 | 9.93268e-05 | 0.0206877 | 3 | weak_win | `exp/integrated/final/cooking-time/best_combined-evaluation/0.toml;exp/integrated/final/cooking-time/best_combined-evaluation/1.toml;exp/integrated/final/cooking-time/best_combined-evaluation/2.toml` | `exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/2` | +| delivery-eta | regression | baseline_plr | baseline_plr | RMSE | lower | mean | 0.55595 | 0.55595 | 0.550226 ± 0.000582292 | 0.550226 ± 0.000582292 | 0 | 0 | 3 | baseline | `exp/integrated/final/delivery-eta/baseline_plr-evaluation/0.toml;exp/integrated/final/delivery-eta/baseline_plr-evaluation/1.toml;exp/integrated/final/delivery-eta/baseline_plr-evaluation/2.toml` | `exp/integrated/final/delivery-eta/baseline_plr-evaluation/0;exp/integrated/final/delivery-eta/baseline_plr-evaluation/1;exp/integrated/final/delivery-eta/baseline_plr-evaluation/2` | +| delivery-eta | regression | best_rla_only | deliv_rla_r4_n0p0001_lr0p5_fr0 | RMSE | lower | best-head | 0.557964 | 0.557868 | 0.55669 ± 0.000859735 | 0.554334 ± 0.00266191 | 0.00235589 | 0.423196 | 3 | clear_win | `exp/integrated/final/delivery-eta/best_rla_only-evaluation/0.toml;exp/integrated/final/delivery-eta/best_rla_only-evaluation/1.toml;exp/integrated/final/delivery-eta/best_rla_only-evaluation/2.toml` | `exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/0;exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/1;exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/2` | +| delivery-eta | regression | best_esam_only | deliv_esam_rho0p001_sf0p25_adapterf | RMSE | lower | mean | 0.55595 | 0.555934 | 0.550226 ± 0.000582292 | 0.551263 ± 0.00129165 | -0.00103689 | -0.188448 | 3 | loss | `exp/integrated/final/delivery-eta/best_esam_only-evaluation/0.toml;exp/integrated/final/delivery-eta/best_esam_only-evaluation/1.toml;exp/integrated/final/delivery-eta/best_esam_only-evaluation/2.toml` | `exp/integrated/final/delivery-eta/best_esam_only-evaluation/0;exp/integrated/final/delivery-eta/best_esam_only-evaluation/1;exp/integrated/final/delivery-eta/best_esam_only-evaluation/2` | +| delivery-eta | regression | best_mfb_only | deliv_mfb_k0p975_sf0p5_numerical_only | RMSE | lower | mean | 0.55595 | 0.556002 | 0.550226 ± 0.000582292 | 0.5502 ± 0.000181893 | 2.62743e-05 | 0.00477518 | 3 | weak_win | `exp/integrated/final/delivery-eta/best_mfb_only-evaluation/0.toml;exp/integrated/final/delivery-eta/best_mfb_only-evaluation/1.toml;exp/integrated/final/delivery-eta/best_mfb_only-evaluation/2.toml` | `exp/integrated/final/delivery-eta/best_mfb_only-evaluation/0;exp/integrated/final/delivery-eta/best_mfb_only-evaluation/1;exp/integrated/final/delivery-eta/best_mfb_only-evaluation/2` | +| delivery-eta | regression | best_cf_fisd_only | deliv_cf_l0p005_consensus_raw | RMSE | lower | greedy-heads | 0.555387 | 0.555387 | 0.551174 ± 0.000706865 | 0.551174 ± 0.000706865 | 0 | 0 | 3 | tie | `exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/0.toml;exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/1.toml;exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/2.toml` | `exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/0;exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/1;exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/2` | +| delivery-eta | regression | best_combined | deliv_comb_rla_cf | RMSE | lower | best-head | 0.557964 | 0.557868 | 0.55669 ± 0.000859735 | 0.554334 ± 0.00266191 | 0.00235589 | 0.423196 | 3 | clear_win | `exp/integrated/final/delivery-eta/best_combined-evaluation/0.toml;exp/integrated/final/delivery-eta/best_combined-evaluation/1.toml;exp/integrated/final/delivery-eta/best_combined-evaluation/2.toml` | `exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/0;exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/1;exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/2` | + +## Rescue validation selections + +| dataset | final variant | selected rescue config | inference | validation metric | matched validation baseline | validation delta | validation status | confirmed | +|---|---|---|---|---:|---:|---:|---|---| +| delivery-eta | best_rla_only | deliv_rla_r4_n0p0001_lr0p5_fr0 | best-head | 0.5572652691682626 | 0.5583881453279098 | 0.0011228761596472348 | validation_win | True | +| delivery-eta | best_esam_only | deliv_esam_rho0p001_sf0p25_adapterf | mean | 0.5558269510040955 | 0.5558749906446844 | 4.8039640588859456e-05 | validation_win | True | +| delivery-eta | best_mfb_only | deliv_mfb_k0p975_sf0p5_numerical_only | mean | 0.5558048331442785 | 0.5558749906446844 | 7.015750040584301e-05 | validation_win | True | +| delivery-eta | best_cf_fisd_only | deliv_cf_l0p005_consensus_raw | greedy-heads | 0.5553348954063525 | 0.5553348954063525 | 0.0 | close_no_validation_win | True | +| delivery-eta | best_combined | deliv_comb_rla_cf | best-head | 0.5572652691682626 | 0.5583881453279098 | 0.0011228761596472348 | validation_win | True | + +## Validation-selected configs + +| dataset | final variant | selected sweep variant | inference | validation metric | +|---|---|---|---|---:| +| sberbank-housing | baseline_plr | baseline_plr | mean | | +| sberbank-housing | best_rla_only | rla_rank2_noise0.001 | greedy-heads | 0.2328613038082034 | +| sberbank-housing | best_esam_only | esam_rho0.0025 | greedy-heads | 0.23295802298295287 | +| sberbank-housing | best_mfb_only | mfb_keep0.8 | greedy-heads | 0.23237596587359466 | +| sberbank-housing | best_cf_fisd_only | cf_fisd_only | greedy-heads | 0.23327114512588593 | +| sberbank-housing | best_combined | sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001 | best-head | 0.23560357747729638 | +| ecom-offers | baseline_plr | baseline_plr | mean | | +| ecom-offers | best_rla_only | rla_rank2_noise0.0001 | best-head | 0.6396541901648656 | +| ecom-offers | best_esam_only | esam_rho0.001 | greedy-heads | 0.641845068859757 | +| ecom-offers | best_mfb_only | mfb_keep0.7 | best-head | 0.6547032702847417 | +| ecom-offers | best_cf_fisd_only | cf_fisd_lambda0.05 | greedy-heads | 0.6419149054445072 | +| ecom-offers | best_combined | mfb_cf_fisd | greedy-heads | 0.6506783465179156 | +| homesite-insurance | baseline_plr | baseline_plr | mean | | +| homesite-insurance | best_rla_only | rla_rank4_noise1e-05 | greedy-heads | 0.9591329763748824 | +| homesite-insurance | best_esam_only | esam_only | greedy-heads | 0.958991133281215 | +| homesite-insurance | best_mfb_only | mfb_keep0.95 | greedy-heads | 0.9592957095194116 | +| homesite-insurance | best_cf_fisd_only | cf_fisd_only | greedy-heads | 0.9594844321379927 | +| homesite-insurance | best_combined | mfb_cf_fisd | greedy-heads | 0.9597570226838343 | +| cooking-time | baseline_plr | baseline_plr | mean | | +| cooking-time | best_rla_only | rla_rank4_noise0.001 | greedy-heads | 0.4624107384246987 | +| cooking-time | best_esam_only | esam_rho0.0025 | greedy-heads | 0.46246732196836127 | +| cooking-time | best_mfb_only | mfb_keep0.8 | greedy-heads | 0.4624822561623469 | +| cooking-time | best_cf_fisd_only | cf_fisd_lambda0.05 | greedy-heads | 0.4628482268025946 | +| cooking-time | best_combined | rla_esam | greedy-heads | 0.4624234187671886 | +| delivery-eta | baseline_plr | baseline_plr | mean | | +| delivery-eta | best_rla_only | deliv_rla_r4_n0p0001_lr0p5_fr0 | best-head | 0.5572652691682626 | +| delivery-eta | best_esam_only | deliv_esam_rho0p001_sf0p25_adapterf | mean | 0.5558269510040955 | +| delivery-eta | best_mfb_only | deliv_mfb_k0p975_sf0p5_numerical_only | mean | 0.5558048331442785 | +| delivery-eta | best_cf_fisd_only | deliv_cf_l0p005_consensus_raw | greedy-heads | 0.5553348954063525 | +| delivery-eta | best_combined | deliv_comb_rla_cf | best-head | 0.5572652691682626 | + +## Module wins vs matched baseline + +- `best_rla_only`: sberbank-housing, delivery-eta +- `best_esam_only`: ecom-offers, homesite-insurance, cooking-time +- `best_mfb_only`: ecom-offers, cooking-time, delivery-eta +- `best_cf_fisd_only`: sberbank-housing, homesite-insurance +- `best_combined`: sberbank-housing, ecom-offers, homesite-insurance, cooking-time, delivery-eta diff --git a/final_report/source_final_integrated_summary.csv b/final_report/source_final_integrated_summary.csv new file mode 100644 index 000000000..c2e9ff847 --- /dev/null +++ b/final_report/source_final_integrated_summary.csv @@ -0,0 +1,31 @@ +dataset,task_type,variant,source_variant,selected_config,metric,direction,metric_direction,validation_metric,matched_validation_baseline,matched_validation_delta,matched_validation_percent_delta,test_metric,mean,std,test_mean,test_std,n_seeds,baseline_mean,baseline_std,matched_baseline_inference_mode,matched_baseline_mean,matched_baseline_std,absolute_delta,percent_delta,precision,inference_mode,selected_inference_mode,config_path,result_path,matched_baseline_result_path,status +sberbank-housing,regression,baseline_plr,baseline_plr,baseline_plr,RMSE,lower,lower,0.23552481981288087,0.23552481981288087,0.0,0.0,0.23441786078206484,0.23441786078206484,0.0035482159973780638,0.23441786078206484,0.0035482159973780638,3,0.23441786078206484,0.0035482159973780638,mean,0.23441786078206484,0.0035482159973780638,0.0,0.0,3 seeds,mean,mean,exp/integrated/final/sberbank-housing/baseline_plr-evaluation/0.toml;exp/integrated/final/sberbank-housing/baseline_plr-evaluation/1.toml;exp/integrated/final/sberbank-housing/baseline_plr-evaluation/2.toml,exp/integrated/final/sberbank-housing/baseline_plr-evaluation/0;exp/integrated/final/sberbank-housing/baseline_plr-evaluation/1;exp/integrated/final/sberbank-housing/baseline_plr-evaluation/2,exp/integrated/final/sberbank-housing/baseline_plr-evaluation/0;exp/integrated/final/sberbank-housing/baseline_plr-evaluation/1;exp/integrated/final/sberbank-housing/baseline_plr-evaluation/2,baseline +sberbank-housing,regression,best_rla_only,rla_rank2_noise0.001,rla_rank2_noise0.001,RMSE,lower,lower,0.23385412997377952,0.23375674885076472,-9.738112301480029e-05,-0.041659170694990484,0.2323679570932143,0.2323679570932143,0.002285510337847783,0.2323679570932143,0.002285510337847783,3,0.2341050589838414,0.002783845646101281,greedy-heads,0.2341050589838414,0.002783845646101281,0.0017371018906270908,0.7420180914360294,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/sberbank-housing/best_rla_only-evaluation/0.toml;exp/integrated/final/sberbank-housing/best_rla_only-evaluation/1.toml;exp/integrated/final/sberbank-housing/best_rla_only-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/2,exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/2,weak_win +sberbank-housing,regression,best_esam_only,esam_rho0.0025,esam_rho0.0025,RMSE,lower,lower,0.23381482399811443,0.23375674885076472,-5.8075147349706535e-05,-0.02484426551756285,0.23440607616465495,0.23440607616465495,0.004011724082624796,0.23440607616465495,0.004011724082624796,3,0.2341050589838414,0.002783845646101281,greedy-heads,0.2341050589838414,0.002783845646101281,-0.00030101718081354534,-0.12858209135682216,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/sberbank-housing/best_esam_only-evaluation/0.toml;exp/integrated/final/sberbank-housing/best_esam_only-evaluation/1.toml;exp/integrated/final/sberbank-housing/best_esam_only-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/2,exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/2,loss +sberbank-housing,regression,best_mfb_only,mfb_keep0.8,mfb_keep0.8,RMSE,lower,lower,0.23321906081613947,0.23375674885076472,0.0005376880346252555,0.23002032551732954,0.23509963234240183,0.23509963234240183,0.003956610803973734,0.23509963234240183,0.003956610803973734,3,0.2341050589838414,0.002783845646101281,greedy-heads,0.2341050589838414,0.002783845646101281,-0.00099457335856043,-0.4248406091169,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/0.toml;exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/1.toml;exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/2,exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/2,loss +sberbank-housing,regression,best_cf_fisd_only,cf_fisd_only,cf_fisd_only,RMSE,lower,lower,0.23407921163352594,0.23375674885076472,-0.00032246278276121254,-0.137948009777925,0.23309306252476805,0.23309306252476805,0.0005850816922811394,0.23309306252476805,0.0005850816922811394,3,0.2341050589838414,0.002783845646101281,greedy-heads,0.2341050589838414,0.002783845646101281,0.00101199645907335,0.4322830371398342,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/0.toml;exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/1.toml;exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/2,exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/2,weak_win +sberbank-housing,regression,best_combined,sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001,sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001,RMSE,lower,lower,0.23887373005966525,0.24017066037702445,0.0012969303173591962,0.5400036437936467,0.24507966333691242,0.24507966333691242,0.008112050773101143,0.24507966333691242,0.008112050773101143,3,0.2502002926581795,0.013600655537586393,best-head,0.2502002926581795,0.013600655537586393,0.005120629321267084,2.0466120430413817,3 seeds,best-head,best-head,exp/integrated/final/sberbank-housing/best_combined-evaluation/0.toml;exp/integrated/final/sberbank-housing/best_combined-evaluation/1.toml;exp/integrated/final/sberbank-housing/best_combined-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/0;exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/1;exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/2,exp/integrated/final/sberbank-housing/baseline_plr-best-head-evaluation/0;exp/integrated/final/sberbank-housing/baseline_plr-best-head-evaluation/1;exp/integrated/final/sberbank-housing/baseline_plr-best-head-evaluation/2,weak_win +ecom-offers,binclass,baseline_plr,baseline_plr,baseline_plr,AUROC,higher,higher,0.6214296509780997,0.6214296509780997,0.0,0.0,0.5900580480375381,0.5900580480375381,0.0009703638630149601,0.5900580480375381,0.0009703638630149601,3,0.5900580480375381,0.0009703638630149601,mean,0.5900580480375381,0.0009703638630149601,0.0,0.0,3 seeds,mean,mean,exp/integrated/final/ecom-offers/baseline_plr-evaluation/0.toml;exp/integrated/final/ecom-offers/baseline_plr-evaluation/1.toml;exp/integrated/final/ecom-offers/baseline_plr-evaluation/2.toml,exp/integrated/final/ecom-offers/baseline_plr-evaluation/0;exp/integrated/final/ecom-offers/baseline_plr-evaluation/1;exp/integrated/final/ecom-offers/baseline_plr-evaluation/2,exp/integrated/final/ecom-offers/baseline_plr-evaluation/0;exp/integrated/final/ecom-offers/baseline_plr-evaluation/1;exp/integrated/final/ecom-offers/baseline_plr-evaluation/2,baseline +ecom-offers,binclass,best_rla_only,rla_rank2_noise0.0001,rla_rank2_noise0.0001,AUROC,higher,higher,0.6348279404183594,0.635345130387201,-0.000517189968841536,-0.08140299564841912,0.5984469268269463,0.5984469268269463,0.002797349140724218,0.5984469268269463,0.002797349140724218,3,0.5996938524587336,0.0035785432553676923,best-head,0.5996938524587336,0.0035785432553676923,-0.001246925631787299,-0.20792703254751188,3 seeds,best-head,best-head,exp/integrated/final/ecom-offers/best_rla_only-evaluation/0.toml;exp/integrated/final/ecom-offers/best_rla_only-evaluation/1.toml;exp/integrated/final/ecom-offers/best_rla_only-evaluation/2.toml,exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/0;exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/1;exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/2,exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/0;exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/1;exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/2,loss +ecom-offers,binclass,best_esam_only,esam_rho0.001,esam_rho0.001,AUROC,higher,higher,0.6366563506545114,0.6365904008411623,6.594981334917538e-05,0.0103598504253335,0.6004196058355985,0.6004196058355985,0.003493353451214683,0.6004196058355985,0.003493353451214683,3,0.6003705999706029,0.0034459818868121214,greedy-heads,0.6003705999706029,0.0034459818868121214,4.900586499556603e-05,0.008162602398912538,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/ecom-offers/best_esam_only-evaluation/0.toml;exp/integrated/final/ecom-offers/best_esam_only-evaluation/1.toml;exp/integrated/final/ecom-offers/best_esam_only-evaluation/2.toml,exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/0;exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/1;exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/2,exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/2,weak_win +ecom-offers,binclass,best_mfb_only,mfb_keep0.7,mfb_keep0.7,AUROC,higher,higher,0.6539769427777475,0.635345130387201,0.018631812390546565,2.9325498063063304,0.6095164790194458,0.6095164790194458,0.0020894834660806928,0.6095164790194458,0.0020894834660806928,3,0.5996938524587336,0.0035785432553676923,best-head,0.5996938524587336,0.0035785432553676923,0.0098226265607122,1.637940179049629,3 seeds,best-head,best-head,exp/integrated/final/ecom-offers/best_mfb_only-evaluation/0.toml;exp/integrated/final/ecom-offers/best_mfb_only-evaluation/1.toml;exp/integrated/final/ecom-offers/best_mfb_only-evaluation/2.toml,exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/0;exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/1;exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/2,exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/0;exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/1;exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/2,clear_win +ecom-offers,binclass,best_cf_fisd_only,cf_fisd_lambda0.05,cf_fisd_lambda0.05,AUROC,higher,higher,0.636440295793426,0.6365904008411623,-0.00015010504773627087,-0.02357953364328597,0.6002048966489212,0.6002048966489212,0.0034582462660955543,0.6002048966489212,0.0034582462660955543,3,0.6003705999706029,0.0034459818868121214,greedy-heads,0.6003705999706029,0.0034459818868121214,-0.00016570332168175383,-0.02760017257505072,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/0.toml;exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/1.toml;exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/0;exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/1;exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/2,exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/2,loss +ecom-offers,binclass,best_combined,mfb_cf_fisd,mfb_cf_fisd,AUROC,higher,higher,0.6485201184226032,0.6365904008411623,0.011929717581440902,1.8740021159096183,0.6052109584242933,0.6052109584242933,0.003222479623952224,0.6052109584242933,0.003222479623952224,3,0.6003705999706029,0.0034459818868121214,greedy-heads,0.6003705999706029,0.0034459818868121214,0.004840358453690352,0.8062284285618516,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/ecom-offers/best_combined-evaluation/0.toml;exp/integrated/final/ecom-offers/best_combined-evaluation/1.toml;exp/integrated/final/ecom-offers/best_combined-evaluation/2.toml,exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/0;exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/1;exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/2,exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/2,clear_win +homesite-insurance,binclass,baseline_plr,baseline_plr,baseline_plr,AUROC,higher,higher,0.9584198249724393,0.9584198249724393,0.0,0.0,0.9624941853379376,0.9624941853379376,0.0001780794332642721,0.9624941853379376,0.0001780794332642721,3,0.9624941853379376,0.0001780794332642721,mean,0.9624941853379376,0.0001780794332642721,0.0,0.0,3 seeds,mean,mean,exp/integrated/final/homesite-insurance/baseline_plr-evaluation/0.toml;exp/integrated/final/homesite-insurance/baseline_plr-evaluation/1.toml;exp/integrated/final/homesite-insurance/baseline_plr-evaluation/2.toml,exp/integrated/final/homesite-insurance/baseline_plr-evaluation/0;exp/integrated/final/homesite-insurance/baseline_plr-evaluation/1;exp/integrated/final/homesite-insurance/baseline_plr-evaluation/2,exp/integrated/final/homesite-insurance/baseline_plr-evaluation/0;exp/integrated/final/homesite-insurance/baseline_plr-evaluation/1;exp/integrated/final/homesite-insurance/baseline_plr-evaluation/2,baseline +homesite-insurance,binclass,best_rla_only,rla_rank4_noise1e-05,rla_rank4_noise1e-05,AUROC,higher,higher,0.9589784960899528,0.9591396202785444,-0.00016112418859159927,-0.01679882523722741,0.9625640466562138,0.9625640466562138,0.0006085287596608143,0.9625640466562138,0.0006085287596608143,3,0.9626755517312511,0.00027672454564782147,greedy-heads,0.9626755517312511,0.00027672454564782147,-0.00011150507503732143,-0.011582830252289419,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/homesite-insurance/best_rla_only-evaluation/0.toml;exp/integrated/final/homesite-insurance/best_rla_only-evaluation/1.toml;exp/integrated/final/homesite-insurance/best_rla_only-evaluation/2.toml,exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/2,exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/2,loss +homesite-insurance,binclass,best_esam_only,esam_only,esam_only,AUROC,higher,higher,0.959080906574986,0.9591396202785444,-5.871370355836625e-05,-0.006121497049753315,0.9627318700697393,0.9627318700697393,0.0004492301463849077,0.9627318700697393,0.0004492301463849077,3,0.9626755517312511,0.00027672454564782147,greedy-heads,0.9626755517312511,0.00027672454564782147,5.6318338488181574e-05,0.005850188922623008,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/homesite-insurance/best_esam_only-evaluation/0.toml;exp/integrated/final/homesite-insurance/best_esam_only-evaluation/1.toml;exp/integrated/final/homesite-insurance/best_esam_only-evaluation/2.toml,exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/2,exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/2,weak_win +homesite-insurance,binclass,best_mfb_only,mfb_keep0.95,mfb_keep0.95,AUROC,higher,higher,0.959182264937648,0.9591396202785444,4.2644659103641125e-05,0.004446136746103415,0.962382329909275,0.962382329909275,0.0004577202193422504,0.962382329909275,0.0004577202193422504,3,0.9626755517312511,0.00027672454564782147,greedy-heads,0.9626755517312511,0.00027672454564782147,-0.00029322182197610847,-0.030459049411692845,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/0.toml;exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/1.toml;exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/2.toml,exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/2,exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/2,loss +homesite-insurance,binclass,best_cf_fisd_only,cf_fisd_only,cf_fisd_only,AUROC,higher,higher,0.959527988245724,0.9591396202785444,0.00038836796717967026,0.04049128604101289,0.9629076428862089,0.9629076428862089,0.00047402677133884024,0.9629076428862089,0.00047402677133884024,3,0.9626755517312511,0.00027672454564782147,greedy-heads,0.9626755517312511,0.00027672454564782147,0.00023209115495781418,0.024108969479948606,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/0.toml;exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/1.toml;exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/2,exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/2,weak_win +homesite-insurance,binclass,best_combined,mfb_cf_fisd,mfb_cf_fisd,AUROC,higher,higher,0.95963296602657,0.9591396202785444,0.0004933457480256198,0.05143628076612526,0.9628490210733115,0.9628490210733115,0.0004379684704485265,0.9628490210733115,0.0004379684704485265,3,0.9626755517312511,0.00027672454564782147,greedy-heads,0.9626755517312511,0.00027672454564782147,0.0001734693420604172,0.018019502183103577,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/homesite-insurance/best_combined-evaluation/0.toml;exp/integrated/final/homesite-insurance/best_combined-evaluation/1.toml;exp/integrated/final/homesite-insurance/best_combined-evaluation/2.toml,exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/2,exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/2,weak_win +cooking-time,regression,baseline_plr,baseline_plr,baseline_plr,RMSE,lower,lower,0.46270875116661525,0.46270875116661525,0.0,0.0,0.4803806470025843,0.4803806470025843,0.00012338540954176603,0.4803806470025843,0.00012338540954176603,3,0.4803806470025843,0.00012338540954176603,mean,0.4803806470025843,0.00012338540954176603,0.0,0.0,3 seeds,mean,mean,exp/integrated/final/cooking-time/baseline_plr-evaluation/0.toml;exp/integrated/final/cooking-time/baseline_plr-evaluation/1.toml;exp/integrated/final/cooking-time/baseline_plr-evaluation/2.toml,exp/integrated/final/cooking-time/baseline_plr-evaluation/0;exp/integrated/final/cooking-time/baseline_plr-evaluation/1;exp/integrated/final/cooking-time/baseline_plr-evaluation/2,exp/integrated/final/cooking-time/baseline_plr-evaluation/0;exp/integrated/final/cooking-time/baseline_plr-evaluation/1;exp/integrated/final/cooking-time/baseline_plr-evaluation/2,baseline +cooking-time,regression,best_rla_only,rla_rank4_noise0.001,rla_rank4_noise0.001,RMSE,lower,lower,0.46244170136784024,0.4624700316244145,2.8330256574282675e-05,0.006125857814996866,0.48014625604758127,0.48014625604758127,0.00016088993704992786,0.48014625604758127,0.00016088993704992786,3,0.48012538218537737,0.0002586745994143849,greedy-heads,0.48012538218537737,0.0002586745994143849,-2.08738622038962e-05,-0.004347585647083486,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/cooking-time/best_rla_only-evaluation/0.toml;exp/integrated/final/cooking-time/best_rla_only-evaluation/1.toml;exp/integrated/final/cooking-time/best_rla_only-evaluation/2.toml,exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/2,exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/2,loss +cooking-time,regression,best_esam_only,esam_rho0.0025,esam_rho0.0025,RMSE,lower,lower,0.46247755253655587,0.4624700316244145,-7.520912141345537e-06,-0.0016262485408899945,0.4800648691653293,0.4800648691653293,8.021206715574865e-05,0.4800648691653293,8.021206715574865e-05,3,0.48012538218537737,0.0002586745994143849,greedy-heads,0.48012538218537737,0.0002586745994143849,6.0513020048058586e-05,0.012603586957353233,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/cooking-time/best_esam_only-evaluation/0.toml;exp/integrated/final/cooking-time/best_esam_only-evaluation/1.toml;exp/integrated/final/cooking-time/best_esam_only-evaluation/2.toml,exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/2,exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/2,weak_win +cooking-time,regression,best_mfb_only,mfb_keep0.8,mfb_keep0.8,RMSE,lower,lower,0.4625031808631155,0.4624700316244145,-3.3149238700957984e-05,-0.007167867415002461,0.4799728414668436,0.4799728414668436,0.00019455249694504936,0.4799728414668436,0.00019455249694504936,3,0.48012538218537737,0.0002586745994143849,greedy-heads,0.48012538218537737,0.0002586745994143849,0.00015254071853376683,0.03177101736205868,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/cooking-time/best_mfb_only-evaluation/0.toml;exp/integrated/final/cooking-time/best_mfb_only-evaluation/1.toml;exp/integrated/final/cooking-time/best_mfb_only-evaluation/2.toml,exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/2,exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/2,weak_win +cooking-time,regression,best_cf_fisd_only,cf_fisd_lambda0.05,cf_fisd_lambda0.05,RMSE,lower,lower,0.46295431080753496,0.4624700316244145,-0.0004842791831204396,-0.10471579778249003,0.48033743363756964,0.48033743363756964,4.649528370692734e-05,0.48033743363756964,4.649528370692734e-05,3,0.48012538218537737,0.0002586745994143849,greedy-heads,0.48012538218537737,0.0002586745994143849,-0.0002120514521922745,-0.04416584918445345,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/0.toml;exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/1.toml;exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/2,exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/2,loss +cooking-time,regression,best_combined,rla_esam,rla_esam,RMSE,lower,lower,0.46244255150399494,0.4624700316244145,2.748012041958736e-05,0.005942032681137007,0.48002605538016,0.48002605538016,9.65539747406379e-05,0.48002605538016,9.65539747406379e-05,3,0.48012538218537737,0.0002586745994143849,greedy-heads,0.48012538218537737,0.0002586745994143849,9.932680521734794e-05,0.020687680531540333,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/cooking-time/best_combined-evaluation/0.toml;exp/integrated/final/cooking-time/best_combined-evaluation/1.toml;exp/integrated/final/cooking-time/best_combined-evaluation/2.toml,exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/2,exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/2,weak_win +delivery-eta,regression,baseline_plr,baseline_plr,baseline_plr,RMSE,lower,lower,0.5559497041253089,0.5559497041253089,0.0,0.0,0.5502258480608934,0.5502258480608934,0.0005822921162605647,0.5502258480608934,0.0005822921162605647,3,0.5502258480608934,0.0005822921162605647,mean,0.5502258480608934,0.0005822921162605647,0.0,0.0,3 seeds,mean,mean,exp/integrated/final/delivery-eta/baseline_plr-evaluation/0.toml;exp/integrated/final/delivery-eta/baseline_plr-evaluation/1.toml;exp/integrated/final/delivery-eta/baseline_plr-evaluation/2.toml,exp/integrated/final/delivery-eta/baseline_plr-evaluation/0;exp/integrated/final/delivery-eta/baseline_plr-evaluation/1;exp/integrated/final/delivery-eta/baseline_plr-evaluation/2,exp/integrated/final/delivery-eta/baseline_plr-evaluation/0;exp/integrated/final/delivery-eta/baseline_plr-evaluation/1;exp/integrated/final/delivery-eta/baseline_plr-evaluation/2,baseline +delivery-eta,regression,best_rla_only,deliv_rla_r4_n0p0001_lr0p5_fr0,deliv_rla_r4_n0p0001_lr0p5_fr0,RMSE,lower,lower,0.557867731128666,0.5579635797458202,9.584861715428161e-05,0.017178292747699653,0.5543336992032142,0.5543336992032142,0.002661909801131988,0.5543336992032142,0.002661909801131988,3,0.5566895883615489,0.000859735360596188,best-head,0.5566895883615489,0.000859735360596188,0.002355889158334734,0.42319619543605924,3 seeds,best-head,best-head,exp/integrated/final/delivery-eta/best_rla_only-evaluation/0.toml;exp/integrated/final/delivery-eta/best_rla_only-evaluation/1.toml;exp/integrated/final/delivery-eta/best_rla_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/0;exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/1;exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/2,exp/integrated/final/delivery-eta/baseline_plr-best-head-evaluation/0;exp/integrated/final/delivery-eta/baseline_plr-best-head-evaluation/1;exp/integrated/final/delivery-eta/baseline_plr-best-head-evaluation/2,clear_win +delivery-eta,regression,best_esam_only,deliv_esam_rho0p001_sf0p25_adapterf,deliv_esam_rho0p001_sf0p25_adapterf,RMSE,lower,lower,0.5559336909117792,0.5559497041253089,1.601321352961982e-05,0.002880334931522961,0.5512627378566571,0.5512627378566571,0.0012916491703925872,0.5512627378566571,0.0012916491703925872,3,0.5502258480608934,0.0005822921162605647,mean,0.5502258480608934,0.0005822921162605647,-0.0010368897957637735,-0.18844803446039146,3 seeds,mean,mean,exp/integrated/final/delivery-eta/best_esam_only-evaluation/0.toml;exp/integrated/final/delivery-eta/best_esam_only-evaluation/1.toml;exp/integrated/final/delivery-eta/best_esam_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_esam_only-evaluation/0;exp/integrated/final/delivery-eta/best_esam_only-evaluation/1;exp/integrated/final/delivery-eta/best_esam_only-evaluation/2,exp/integrated/final/delivery-eta/baseline_plr-evaluation/0;exp/integrated/final/delivery-eta/baseline_plr-evaluation/1;exp/integrated/final/delivery-eta/baseline_plr-evaluation/2,loss +delivery-eta,regression,best_mfb_only,deliv_mfb_k0p975_sf0p5_numerical_only,deliv_mfb_k0p975_sf0p5_numerical_only,RMSE,lower,lower,0.5560023662408545,0.5559497041253089,-5.2662115545665245e-05,-0.009472460396128821,0.5501995737833563,0.5501995737833563,0.0001818929992957081,0.5501995737833563,0.0001818929992957081,3,0.5502258480608934,0.0005822921162605647,mean,0.5502258480608934,0.0005822921162605647,2.6274277537097213e-05,0.004775180524450652,3 seeds,mean,mean,exp/integrated/final/delivery-eta/best_mfb_only-evaluation/0.toml;exp/integrated/final/delivery-eta/best_mfb_only-evaluation/1.toml;exp/integrated/final/delivery-eta/best_mfb_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_mfb_only-evaluation/0;exp/integrated/final/delivery-eta/best_mfb_only-evaluation/1;exp/integrated/final/delivery-eta/best_mfb_only-evaluation/2,exp/integrated/final/delivery-eta/baseline_plr-evaluation/0;exp/integrated/final/delivery-eta/baseline_plr-evaluation/1;exp/integrated/final/delivery-eta/baseline_plr-evaluation/2,weak_win +delivery-eta,regression,best_cf_fisd_only,deliv_cf_l0p005_consensus_raw,deliv_cf_l0p005_consensus_raw,RMSE,lower,lower,0.5553874207148172,0.5553874207148172,0.0,0.0,0.5511744515458483,0.5511744515458483,0.0007068650190236373,0.5511744515458483,0.0007068650190236373,3,0.5511744515458483,0.0007068650190236373,greedy-heads,0.5511744515458483,0.0007068650190236373,0.0,0.0,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/0.toml;exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/1.toml;exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/0;exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/1;exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/2,exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/2,tie +delivery-eta,regression,best_combined,deliv_comb_rla_cf,deliv_comb_rla_cf,RMSE,lower,lower,0.557867731128666,0.5579635797458202,9.584861715428161e-05,0.017178292747699653,0.5543336992032142,0.5543336992032142,0.002661909801131988,0.5543336992032142,0.002661909801131988,3,0.5566895883615489,0.000859735360596188,best-head,0.5566895883615489,0.000859735360596188,0.002355889158334734,0.42319619543605924,3 seeds,best-head,best-head,exp/integrated/final/delivery-eta/best_combined-evaluation/0.toml;exp/integrated/final/delivery-eta/best_combined-evaluation/1.toml;exp/integrated/final/delivery-eta/best_combined-evaluation/2.toml,exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/0;exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/1;exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/2,exp/integrated/final/delivery-eta/baseline_plr-best-head-evaluation/0;exp/integrated/final/delivery-eta/baseline_plr-best-head-evaluation/1;exp/integrated/final/delivery-eta/baseline_plr-best-head-evaluation/2,clear_win diff --git a/tabm_integrated/FINAL_EXPERIMENT_REPORT.md b/tabm_integrated/FINAL_EXPERIMENT_REPORT.md new file mode 100644 index 000000000..c11fe9602 --- /dev/null +++ b/tabm_integrated/FINAL_EXPERIMENT_REPORT.md @@ -0,0 +1,116 @@ +# Integrated TabM final experiment report + +Official TabM PLR/PiecewiseLinearEmbeddings baseline configs are preserved exactly; variants add only module flags for RLA, ESAM, MFB, and CF-FISD. +Deltas and win/loss statuses are computed against `baseline_plr` using the same selected inference mode (`mean`, `best-head`, or `greedy-heads`) as the variant row. + +## Matched-inference correction + +The earlier mean-baseline comparison changed when every variant was compared with the matching baseline inference mode. Rescue sweeps are selected against matched validation baselines only. + +| dataset | variant | inference | mean-baseline status | matched-baseline status | mean-baseline delta | matched delta | +|---|---|---|---|---|---:|---:| +| sberbank-housing | best_esam_only | greedy-heads | weak_win | loss | 1.17846e-05 | -0.000301017 | +| sberbank-housing | best_combined | best-head | loss | weak_win | -0.0106618 | 0.00512063 | +| ecom-offers | best_rla_only | best-head | clear_win | loss | 0.00838888 | -0.00124693 | +| ecom-offers | best_esam_only | greedy-heads | clear_win | weak_win | 0.0103616 | 4.90059e-05 | +| ecom-offers | best_cf_fisd_only | greedy-heads | clear_win | loss | 0.0101468 | -0.000165703 | +| homesite-insurance | best_rla_only | greedy-heads | weak_win | loss | 6.98613e-05 | -0.000111505 | +| homesite-insurance | best_esam_only | greedy-heads | clear_win | weak_win | 0.000237685 | 5.63183e-05 | +| homesite-insurance | best_cf_fisd_only | greedy-heads | clear_win | weak_win | 0.000413458 | 0.000232091 | +| homesite-insurance | best_combined | greedy-heads | clear_win | weak_win | 0.000354836 | 0.000173469 | +| cooking-time | best_rla_only | greedy-heads | clear_win | loss | 0.000234391 | -2.08739e-05 | +| cooking-time | best_esam_only | greedy-heads | clear_win | weak_win | 0.000315778 | 6.0513e-05 | +| cooking-time | best_mfb_only | greedy-heads | clear_win | weak_win | 0.000407806 | 0.000152541 | +| cooking-time | best_cf_fisd_only | greedy-heads | weak_win | loss | 4.32134e-05 | -0.000212051 | +| cooking-time | best_combined | greedy-heads | clear_win | weak_win | 0.000354592 | 9.93268e-05 | +| delivery-eta | best_rla_only | best-head | loss | clear_win | -0.00410785 | 0.00235589 | +| delivery-eta | best_cf_fisd_only | greedy-heads | loss | tie | -0.000948603 | 0 | +| delivery-eta | best_combined | best-head | loss | clear_win | -0.00410785 | 0.00235589 | + +## Final 3-seed results + +| dataset | task | variant | selected config | metric | direction | inference | matched validation baseline | validation metric | matched baseline test mean ± std | result test mean ± std | delta | % delta | n | status | config path | result path | +|---|---|---|---|---|---|---|---:|---:|---:|---:|---:|---:|---:|---|---|---| +| sberbank-housing | regression | baseline_plr | baseline_plr | RMSE | lower | mean | 0.235525 | 0.235525 | 0.234418 ± 0.00354822 | 0.234418 ± 0.00354822 | 0 | 0 | 3 | baseline | `exp/integrated/final/sberbank-housing/baseline_plr-evaluation/0.toml;exp/integrated/final/sberbank-housing/baseline_plr-evaluation/1.toml;exp/integrated/final/sberbank-housing/baseline_plr-evaluation/2.toml` | `exp/integrated/final/sberbank-housing/baseline_plr-evaluation/0;exp/integrated/final/sberbank-housing/baseline_plr-evaluation/1;exp/integrated/final/sberbank-housing/baseline_plr-evaluation/2` | +| sberbank-housing | regression | best_rla_only | rla_rank2_noise0.001 | RMSE | lower | greedy-heads | 0.233757 | 0.233854 | 0.234105 ± 0.00278385 | 0.232368 ± 0.00228551 | 0.0017371 | 0.742018 | 3 | weak_win | `exp/integrated/final/sberbank-housing/best_rla_only-evaluation/0.toml;exp/integrated/final/sberbank-housing/best_rla_only-evaluation/1.toml;exp/integrated/final/sberbank-housing/best_rla_only-evaluation/2.toml` | `exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/2` | +| sberbank-housing | regression | best_esam_only | esam_rho0.0025 | RMSE | lower | greedy-heads | 0.233757 | 0.233815 | 0.234105 ± 0.00278385 | 0.234406 ± 0.00401172 | -0.000301017 | -0.128582 | 3 | loss | `exp/integrated/final/sberbank-housing/best_esam_only-evaluation/0.toml;exp/integrated/final/sberbank-housing/best_esam_only-evaluation/1.toml;exp/integrated/final/sberbank-housing/best_esam_only-evaluation/2.toml` | `exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/2` | +| sberbank-housing | regression | best_mfb_only | mfb_keep0.8 | RMSE | lower | greedy-heads | 0.233757 | 0.233219 | 0.234105 ± 0.00278385 | 0.2351 ± 0.00395661 | -0.000994573 | -0.424841 | 3 | loss | `exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/0.toml;exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/1.toml;exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/2.toml` | `exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/2` | +| sberbank-housing | regression | best_cf_fisd_only | cf_fisd_only | RMSE | lower | greedy-heads | 0.233757 | 0.234079 | 0.234105 ± 0.00278385 | 0.233093 ± 0.000585082 | 0.001012 | 0.432283 | 3 | weak_win | `exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/0.toml;exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/1.toml;exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/2.toml` | `exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/2` | +| sberbank-housing | regression | best_combined | sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001 | RMSE | lower | best-head | 0.240171 | 0.238874 | 0.2502 ± 0.0136007 | 0.24508 ± 0.00811205 | 0.00512063 | 2.04661 | 3 | weak_win | `exp/integrated/final/sberbank-housing/best_combined-evaluation/0.toml;exp/integrated/final/sberbank-housing/best_combined-evaluation/1.toml;exp/integrated/final/sberbank-housing/best_combined-evaluation/2.toml` | `exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/0;exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/1;exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/2` | +| ecom-offers | binclass | baseline_plr | baseline_plr | AUROC | higher | mean | 0.62143 | 0.62143 | 0.590058 ± 0.000970364 | 0.590058 ± 0.000970364 | 0 | 0 | 3 | baseline | `exp/integrated/final/ecom-offers/baseline_plr-evaluation/0.toml;exp/integrated/final/ecom-offers/baseline_plr-evaluation/1.toml;exp/integrated/final/ecom-offers/baseline_plr-evaluation/2.toml` | `exp/integrated/final/ecom-offers/baseline_plr-evaluation/0;exp/integrated/final/ecom-offers/baseline_plr-evaluation/1;exp/integrated/final/ecom-offers/baseline_plr-evaluation/2` | +| ecom-offers | binclass | best_rla_only | rla_rank2_noise0.0001 | AUROC | higher | best-head | 0.635345 | 0.634828 | 0.599694 ± 0.00357854 | 0.598447 ± 0.00279735 | -0.00124693 | -0.207927 | 3 | loss | `exp/integrated/final/ecom-offers/best_rla_only-evaluation/0.toml;exp/integrated/final/ecom-offers/best_rla_only-evaluation/1.toml;exp/integrated/final/ecom-offers/best_rla_only-evaluation/2.toml` | `exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/0;exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/1;exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/2` | +| ecom-offers | binclass | best_esam_only | esam_rho0.001 | AUROC | higher | greedy-heads | 0.63659 | 0.636656 | 0.600371 ± 0.00344598 | 0.60042 ± 0.00349335 | 4.90059e-05 | 0.0081626 | 3 | weak_win | `exp/integrated/final/ecom-offers/best_esam_only-evaluation/0.toml;exp/integrated/final/ecom-offers/best_esam_only-evaluation/1.toml;exp/integrated/final/ecom-offers/best_esam_only-evaluation/2.toml` | `exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/0;exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/1;exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/2` | +| ecom-offers | binclass | best_mfb_only | mfb_keep0.7 | AUROC | higher | best-head | 0.635345 | 0.653977 | 0.599694 ± 0.00357854 | 0.609516 ± 0.00208948 | 0.00982263 | 1.63794 | 3 | clear_win | `exp/integrated/final/ecom-offers/best_mfb_only-evaluation/0.toml;exp/integrated/final/ecom-offers/best_mfb_only-evaluation/1.toml;exp/integrated/final/ecom-offers/best_mfb_only-evaluation/2.toml` | `exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/0;exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/1;exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/2` | +| ecom-offers | binclass | best_cf_fisd_only | cf_fisd_lambda0.05 | AUROC | higher | greedy-heads | 0.63659 | 0.63644 | 0.600371 ± 0.00344598 | 0.600205 ± 0.00345825 | -0.000165703 | -0.0276002 | 3 | loss | `exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/0.toml;exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/1.toml;exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/2.toml` | `exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/0;exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/1;exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/2` | +| ecom-offers | binclass | best_combined | mfb_cf_fisd | AUROC | higher | greedy-heads | 0.63659 | 0.64852 | 0.600371 ± 0.00344598 | 0.605211 ± 0.00322248 | 0.00484036 | 0.806228 | 3 | clear_win | `exp/integrated/final/ecom-offers/best_combined-evaluation/0.toml;exp/integrated/final/ecom-offers/best_combined-evaluation/1.toml;exp/integrated/final/ecom-offers/best_combined-evaluation/2.toml` | `exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/0;exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/1;exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/2` | +| homesite-insurance | binclass | baseline_plr | baseline_plr | AUROC | higher | mean | 0.95842 | 0.95842 | 0.962494 ± 0.000178079 | 0.962494 ± 0.000178079 | 0 | 0 | 3 | baseline | `exp/integrated/final/homesite-insurance/baseline_plr-evaluation/0.toml;exp/integrated/final/homesite-insurance/baseline_plr-evaluation/1.toml;exp/integrated/final/homesite-insurance/baseline_plr-evaluation/2.toml` | `exp/integrated/final/homesite-insurance/baseline_plr-evaluation/0;exp/integrated/final/homesite-insurance/baseline_plr-evaluation/1;exp/integrated/final/homesite-insurance/baseline_plr-evaluation/2` | +| homesite-insurance | binclass | best_rla_only | rla_rank4_noise1e-05 | AUROC | higher | greedy-heads | 0.95914 | 0.958978 | 0.962676 ± 0.000276725 | 0.962564 ± 0.000608529 | -0.000111505 | -0.0115828 | 3 | loss | `exp/integrated/final/homesite-insurance/best_rla_only-evaluation/0.toml;exp/integrated/final/homesite-insurance/best_rla_only-evaluation/1.toml;exp/integrated/final/homesite-insurance/best_rla_only-evaluation/2.toml` | `exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/2` | +| homesite-insurance | binclass | best_esam_only | esam_only | AUROC | higher | greedy-heads | 0.95914 | 0.959081 | 0.962676 ± 0.000276725 | 0.962732 ± 0.00044923 | 5.63183e-05 | 0.00585019 | 3 | weak_win | `exp/integrated/final/homesite-insurance/best_esam_only-evaluation/0.toml;exp/integrated/final/homesite-insurance/best_esam_only-evaluation/1.toml;exp/integrated/final/homesite-insurance/best_esam_only-evaluation/2.toml` | `exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/2` | +| homesite-insurance | binclass | best_mfb_only | mfb_keep0.95 | AUROC | higher | greedy-heads | 0.95914 | 0.959182 | 0.962676 ± 0.000276725 | 0.962382 ± 0.00045772 | -0.000293222 | -0.030459 | 3 | loss | `exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/0.toml;exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/1.toml;exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/2.toml` | `exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/2` | +| homesite-insurance | binclass | best_cf_fisd_only | cf_fisd_only | AUROC | higher | greedy-heads | 0.95914 | 0.959528 | 0.962676 ± 0.000276725 | 0.962908 ± 0.000474027 | 0.000232091 | 0.024109 | 3 | weak_win | `exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/0.toml;exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/1.toml;exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/2.toml` | `exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/2` | +| homesite-insurance | binclass | best_combined | mfb_cf_fisd | AUROC | higher | greedy-heads | 0.95914 | 0.959633 | 0.962676 ± 0.000276725 | 0.962849 ± 0.000437968 | 0.000173469 | 0.0180195 | 3 | weak_win | `exp/integrated/final/homesite-insurance/best_combined-evaluation/0.toml;exp/integrated/final/homesite-insurance/best_combined-evaluation/1.toml;exp/integrated/final/homesite-insurance/best_combined-evaluation/2.toml` | `exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/2` | +| cooking-time | regression | baseline_plr | baseline_plr | RMSE | lower | mean | 0.462709 | 0.462709 | 0.480381 ± 0.000123385 | 0.480381 ± 0.000123385 | 0 | 0 | 3 | baseline | `exp/integrated/final/cooking-time/baseline_plr-evaluation/0.toml;exp/integrated/final/cooking-time/baseline_plr-evaluation/1.toml;exp/integrated/final/cooking-time/baseline_plr-evaluation/2.toml` | `exp/integrated/final/cooking-time/baseline_plr-evaluation/0;exp/integrated/final/cooking-time/baseline_plr-evaluation/1;exp/integrated/final/cooking-time/baseline_plr-evaluation/2` | +| cooking-time | regression | best_rla_only | rla_rank4_noise0.001 | RMSE | lower | greedy-heads | 0.46247 | 0.462442 | 0.480125 ± 0.000258675 | 0.480146 ± 0.00016089 | -2.08739e-05 | -0.00434759 | 3 | loss | `exp/integrated/final/cooking-time/best_rla_only-evaluation/0.toml;exp/integrated/final/cooking-time/best_rla_only-evaluation/1.toml;exp/integrated/final/cooking-time/best_rla_only-evaluation/2.toml` | `exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/2` | +| cooking-time | regression | best_esam_only | esam_rho0.0025 | RMSE | lower | greedy-heads | 0.46247 | 0.462478 | 0.480125 ± 0.000258675 | 0.480065 ± 8.02121e-05 | 6.0513e-05 | 0.0126036 | 3 | weak_win | `exp/integrated/final/cooking-time/best_esam_only-evaluation/0.toml;exp/integrated/final/cooking-time/best_esam_only-evaluation/1.toml;exp/integrated/final/cooking-time/best_esam_only-evaluation/2.toml` | `exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/2` | +| cooking-time | regression | best_mfb_only | mfb_keep0.8 | RMSE | lower | greedy-heads | 0.46247 | 0.462503 | 0.480125 ± 0.000258675 | 0.479973 ± 0.000194552 | 0.000152541 | 0.031771 | 3 | weak_win | `exp/integrated/final/cooking-time/best_mfb_only-evaluation/0.toml;exp/integrated/final/cooking-time/best_mfb_only-evaluation/1.toml;exp/integrated/final/cooking-time/best_mfb_only-evaluation/2.toml` | `exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/2` | +| cooking-time | regression | best_cf_fisd_only | cf_fisd_lambda0.05 | RMSE | lower | greedy-heads | 0.46247 | 0.462954 | 0.480125 ± 0.000258675 | 0.480337 ± 4.64953e-05 | -0.000212051 | -0.0441658 | 3 | loss | `exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/0.toml;exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/1.toml;exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/2.toml` | `exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/2` | +| cooking-time | regression | best_combined | rla_esam | RMSE | lower | greedy-heads | 0.46247 | 0.462443 | 0.480125 ± 0.000258675 | 0.480026 ± 9.6554e-05 | 9.93268e-05 | 0.0206877 | 3 | weak_win | `exp/integrated/final/cooking-time/best_combined-evaluation/0.toml;exp/integrated/final/cooking-time/best_combined-evaluation/1.toml;exp/integrated/final/cooking-time/best_combined-evaluation/2.toml` | `exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/2` | +| delivery-eta | regression | baseline_plr | baseline_plr | RMSE | lower | mean | 0.55595 | 0.55595 | 0.550226 ± 0.000582292 | 0.550226 ± 0.000582292 | 0 | 0 | 3 | baseline | `exp/integrated/final/delivery-eta/baseline_plr-evaluation/0.toml;exp/integrated/final/delivery-eta/baseline_plr-evaluation/1.toml;exp/integrated/final/delivery-eta/baseline_plr-evaluation/2.toml` | `exp/integrated/final/delivery-eta/baseline_plr-evaluation/0;exp/integrated/final/delivery-eta/baseline_plr-evaluation/1;exp/integrated/final/delivery-eta/baseline_plr-evaluation/2` | +| delivery-eta | regression | best_rla_only | deliv_rla_r4_n0p0001_lr0p5_fr0 | RMSE | lower | best-head | 0.557964 | 0.557868 | 0.55669 ± 0.000859735 | 0.554334 ± 0.00266191 | 0.00235589 | 0.423196 | 3 | clear_win | `exp/integrated/final/delivery-eta/best_rla_only-evaluation/0.toml;exp/integrated/final/delivery-eta/best_rla_only-evaluation/1.toml;exp/integrated/final/delivery-eta/best_rla_only-evaluation/2.toml` | `exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/0;exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/1;exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/2` | +| delivery-eta | regression | best_esam_only | deliv_esam_rho0p001_sf0p25_adapterf | RMSE | lower | mean | 0.55595 | 0.555934 | 0.550226 ± 0.000582292 | 0.551263 ± 0.00129165 | -0.00103689 | -0.188448 | 3 | loss | `exp/integrated/final/delivery-eta/best_esam_only-evaluation/0.toml;exp/integrated/final/delivery-eta/best_esam_only-evaluation/1.toml;exp/integrated/final/delivery-eta/best_esam_only-evaluation/2.toml` | `exp/integrated/final/delivery-eta/best_esam_only-evaluation/0;exp/integrated/final/delivery-eta/best_esam_only-evaluation/1;exp/integrated/final/delivery-eta/best_esam_only-evaluation/2` | +| delivery-eta | regression | best_mfb_only | deliv_mfb_k0p975_sf0p5_numerical_only | RMSE | lower | mean | 0.55595 | 0.556002 | 0.550226 ± 0.000582292 | 0.5502 ± 0.000181893 | 2.62743e-05 | 0.00477518 | 3 | weak_win | `exp/integrated/final/delivery-eta/best_mfb_only-evaluation/0.toml;exp/integrated/final/delivery-eta/best_mfb_only-evaluation/1.toml;exp/integrated/final/delivery-eta/best_mfb_only-evaluation/2.toml` | `exp/integrated/final/delivery-eta/best_mfb_only-evaluation/0;exp/integrated/final/delivery-eta/best_mfb_only-evaluation/1;exp/integrated/final/delivery-eta/best_mfb_only-evaluation/2` | +| delivery-eta | regression | best_cf_fisd_only | deliv_cf_l0p005_consensus_raw | RMSE | lower | greedy-heads | 0.555387 | 0.555387 | 0.551174 ± 0.000706865 | 0.551174 ± 0.000706865 | 0 | 0 | 3 | tie | `exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/0.toml;exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/1.toml;exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/2.toml` | `exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/0;exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/1;exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/2` | +| delivery-eta | regression | best_combined | deliv_comb_rla_cf | RMSE | lower | best-head | 0.557964 | 0.557868 | 0.55669 ± 0.000859735 | 0.554334 ± 0.00266191 | 0.00235589 | 0.423196 | 3 | clear_win | `exp/integrated/final/delivery-eta/best_combined-evaluation/0.toml;exp/integrated/final/delivery-eta/best_combined-evaluation/1.toml;exp/integrated/final/delivery-eta/best_combined-evaluation/2.toml` | `exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/0;exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/1;exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/2` | + +## Rescue validation selections + +| dataset | final variant | selected rescue config | inference | validation metric | matched validation baseline | validation delta | validation status | confirmed | +|---|---|---|---|---:|---:|---:|---|---| +| delivery-eta | best_rla_only | deliv_rla_r4_n0p0001_lr0p5_fr0 | best-head | 0.5572652691682626 | 0.5583881453279098 | 0.0011228761596472348 | validation_win | True | +| delivery-eta | best_esam_only | deliv_esam_rho0p001_sf0p25_adapterf | mean | 0.5558269510040955 | 0.5558749906446844 | 4.8039640588859456e-05 | validation_win | True | +| delivery-eta | best_mfb_only | deliv_mfb_k0p975_sf0p5_numerical_only | mean | 0.5558048331442785 | 0.5558749906446844 | 7.015750040584301e-05 | validation_win | True | +| delivery-eta | best_cf_fisd_only | deliv_cf_l0p005_consensus_raw | greedy-heads | 0.5553348954063525 | 0.5553348954063525 | 0.0 | close_no_validation_win | True | +| delivery-eta | best_combined | deliv_comb_rla_cf | best-head | 0.5572652691682626 | 0.5583881453279098 | 0.0011228761596472348 | validation_win | True | + +## Validation-selected configs + +| dataset | final variant | selected sweep variant | inference | validation metric | +|---|---|---|---|---:| +| sberbank-housing | baseline_plr | baseline_plr | mean | | +| sberbank-housing | best_rla_only | rla_rank2_noise0.001 | greedy-heads | 0.2328613038082034 | +| sberbank-housing | best_esam_only | esam_rho0.0025 | greedy-heads | 0.23295802298295287 | +| sberbank-housing | best_mfb_only | mfb_keep0.8 | greedy-heads | 0.23237596587359466 | +| sberbank-housing | best_cf_fisd_only | cf_fisd_only | greedy-heads | 0.23327114512588593 | +| sberbank-housing | best_combined | sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001 | best-head | 0.23560357747729638 | +| ecom-offers | baseline_plr | baseline_plr | mean | | +| ecom-offers | best_rla_only | rla_rank2_noise0.0001 | best-head | 0.6396541901648656 | +| ecom-offers | best_esam_only | esam_rho0.001 | greedy-heads | 0.641845068859757 | +| ecom-offers | best_mfb_only | mfb_keep0.7 | best-head | 0.6547032702847417 | +| ecom-offers | best_cf_fisd_only | cf_fisd_lambda0.05 | greedy-heads | 0.6419149054445072 | +| ecom-offers | best_combined | mfb_cf_fisd | greedy-heads | 0.6506783465179156 | +| homesite-insurance | baseline_plr | baseline_plr | mean | | +| homesite-insurance | best_rla_only | rla_rank4_noise1e-05 | greedy-heads | 0.9591329763748824 | +| homesite-insurance | best_esam_only | esam_only | greedy-heads | 0.958991133281215 | +| homesite-insurance | best_mfb_only | mfb_keep0.95 | greedy-heads | 0.9592957095194116 | +| homesite-insurance | best_cf_fisd_only | cf_fisd_only | greedy-heads | 0.9594844321379927 | +| homesite-insurance | best_combined | mfb_cf_fisd | greedy-heads | 0.9597570226838343 | +| cooking-time | baseline_plr | baseline_plr | mean | | +| cooking-time | best_rla_only | rla_rank4_noise0.001 | greedy-heads | 0.4624107384246987 | +| cooking-time | best_esam_only | esam_rho0.0025 | greedy-heads | 0.46246732196836127 | +| cooking-time | best_mfb_only | mfb_keep0.8 | greedy-heads | 0.4624822561623469 | +| cooking-time | best_cf_fisd_only | cf_fisd_lambda0.05 | greedy-heads | 0.4628482268025946 | +| cooking-time | best_combined | rla_esam | greedy-heads | 0.4624234187671886 | +| delivery-eta | baseline_plr | baseline_plr | mean | | +| delivery-eta | best_rla_only | deliv_rla_r4_n0p0001_lr0p5_fr0 | best-head | 0.5572652691682626 | +| delivery-eta | best_esam_only | deliv_esam_rho0p001_sf0p25_adapterf | mean | 0.5558269510040955 | +| delivery-eta | best_mfb_only | deliv_mfb_k0p975_sf0p5_numerical_only | mean | 0.5558048331442785 | +| delivery-eta | best_cf_fisd_only | deliv_cf_l0p005_consensus_raw | greedy-heads | 0.5553348954063525 | +| delivery-eta | best_combined | deliv_comb_rla_cf | best-head | 0.5572652691682626 | + +## Module wins vs matched baseline + +- `best_rla_only`: sberbank-housing, delivery-eta +- `best_esam_only`: ecom-offers, homesite-insurance, cooking-time +- `best_mfb_only`: ecom-offers, cooking-time, delivery-eta +- `best_cf_fisd_only`: sberbank-housing, homesite-insurance +- `best_combined`: sberbank-housing, ecom-offers, homesite-insurance, cooking-time, delivery-eta diff --git a/tabm_integrated/__init__.py b/tabm_integrated/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/.gitignore b/tabm_integrated/paper/.gitignore new file mode 100644 index 000000000..48d800d9e --- /dev/null +++ b/tabm_integrated/paper/.gitignore @@ -0,0 +1,5 @@ +cache/ +__pycache__/ +.pytest_cache/ +*.pyc +cache diff --git a/tabm_integrated/paper/bin/run_integrated.py b/tabm_integrated/paper/bin/run_integrated.py new file mode 100644 index 000000000..006f440ed --- /dev/null +++ b/tabm_integrated/paper/bin/run_integrated.py @@ -0,0 +1,1707 @@ +import math +import json +import shutil +import statistics +import sys +from pathlib import Path +from typing import Any, Literal + +import delu +import numpy as np +import rtdl_num_embeddings +import scipy +import torch +import torch.nn as nn +import torch.utils.tensorboard +from loguru import logger +from torch import Tensor +from tqdm import tqdm +from typing_extensions import NotRequired, TypedDict + +if __name__ == '__main__': + _cwd = Path.cwd() + assert _cwd.joinpath( + 'pixi.toml' + ).exists(), 'The script must be run from the `paper/` directory' + sys.path.append(str(_cwd)) + del _cwd + +import lib +import lib.data +import lib.deep +import lib.env +import lib.cf_fisd +from lib import KWArgs, PartKey + + +def _get_first_ensemble_layer( + backbone: lib.deep.MLP, +) -> lib.deep.LinearEfficientEnsemble: + if isinstance(backbone, lib.deep.MLP): + return backbone.blocks[0][0] # type: ignore[code] + else: + raise RuntimeError(f'Unsupported backbone: {backbone}') + + +@torch.inference_mode() +def _init_first_adapter( + weight: Tensor, + distribution: Literal['normal', 'random-signs'], + init_sections: list[int], +) -> None: + """Initialize the first adapter. + + NOTE + The `init_sections` argument is a historical artifact that accidentally leaked + from irrelevant experiments to the final models. Perhaps, the code related + to `init_sections` can be simply removed, but this was not tested. + """ + assert weight.ndim == 2 + assert weight.shape[1] == sum(init_sections) + + if distribution == 'normal': + init_fn_ = nn.init.normal_ + elif distribution == 'random-signs': + init_fn_ = lib.deep.init_random_signs_ + else: + raise ValueError(f'Unknown distribution: {distribution}') + + section_bounds = [0, *torch.tensor(init_sections).cumsum(0).tolist()] + for i in range(len(init_sections)): + # NOTE + # As noted above, this section-based initialization is an arbitrary historical + # artifact. Consider the first adapter of one ensemble member. + # This adapter vector is implicitly split into "sections", + # where one section corresponds to one feature. The code below ensures that + # the adapter weights in one section are initialized with the same random value + # from the given distribution. + w = torch.empty((len(weight), 1), dtype=weight.dtype, device=weight.device) + init_fn_(w) + weight[:, section_bounds[i] : section_bounds[i + 1]] = w + + +def _get_first_rankr_layer( + backbone: lib.deep.MLP, +) -> lib.deep.LinearEfficientEnsembleRankR: + if isinstance(backbone, lib.deep.MLP): + return backbone.blocks[0][0] # type: ignore[code] + raise RuntimeError(f'Unsupported backbone: {backbone}') + + +@torch.inference_mode() +def _init_first_adapter_rankr( + R: Tensor, + distribution: Literal['normal', 'random-signs'], + init_sections: list[int], +) -> None: + """Initialise the first-layer R parameter (k, d_in, rank). + + Each rank column receives an independent section-wise random sign + (or normal) draw, then the entire R tensor is scaled by 1/sqrt(rank) + so the sum over the rank axis preserves the variance of the rank-1 + baseline at initialisation. + """ + assert R.ndim == 3 + k, d_in, rank = R.shape + assert d_in == sum(init_sections) + + if distribution == 'normal': + init_fn_ = nn.init.normal_ + elif distribution == 'random-signs': + init_fn_ = lib.deep.init_random_signs_ + else: + raise ValueError(f'Unknown distribution: {distribution}') + + section_bounds = [0, *torch.tensor(init_sections).cumsum(0).tolist()] + for j in range(rank): + for i in range(len(init_sections)): + w = torch.empty((k, 1), dtype=R.dtype, device=R.device) + init_fn_(w) + R[:, section_bounds[i] : section_bounds[i + 1], j] = w + R.mul_(rank**-0.5) + + +def _replace_first_with_rankr( + backbone: lib.deep.MLP, + *, + k: int, + rank: int, + additive: bool, + init_mode: str = 'variance_preserving', + base_preserve_noise: float = 1e-3, +) -> None: + """Replace the first ensemble linear layer with the rank-r variant. + + The remaining layers stay as the baseline rank-1 class. This implements + the RLA-first variant from Section 3.1 of the spec. + + NOTE on RNG isolation: ``LinearEfficientEnsembleRankR.__init__`` calls + ``reset_parameters`` which consumes global RNG for its (about-to-be + overwritten) weight and bias initialisation. Without isolation, that + extra draw drifts the RNG state and ``_init_first_adapter_basepreserve`` + below would then sample a different sequence of random signs than the + baseline construction did, breaking full-model bit-equivalence at + init. Save+restore the global RNG state around the construction. + """ + first = _get_first_ensemble_layer(backbone) + rng_state = torch.random.get_rng_state() + try: + new = lib.deep.LinearEfficientEnsembleRankR( + in_features=first.in_features, + out_features=first.out_features, + bias=first.bias is not None, + k=k, + rank=rank, + scaling_init='ones', + additive=additive, + init_mode=init_mode, + base_preserve_noise=base_preserve_noise, + ) + finally: + torch.random.set_rng_state(rng_state) + # Re-use the same backbone weights for stability of the comparison. + with torch.inference_mode(): + new.weight.copy_(first.weight) + if first.bias is not None and new.bias is not None: + new.bias.copy_(first.bias) + # Splice into the backbone (paper/lib MLP block 0 sequential[0]). + backbone.blocks[0][0] = new # type: ignore[code] + + +@torch.inference_mode() +def _init_first_adapter_basepreserve( + R: Tensor, + distribution: Literal['normal', 'random-signs'], + init_sections: list[int], +) -> None: + """Re-initialise *only path 0* of a rank-r R parameter using the + section-based scheme. Extra paths (j >= 1) are left untouched, + preserving the base-preserving zero/tiny-noise init. + """ + assert R.ndim == 3 + k, d_in, rank = R.shape + assert d_in == sum(init_sections) + + if distribution == 'normal': + init_fn_ = nn.init.normal_ + elif distribution == 'random-signs': + init_fn_ = lib.deep.init_random_signs_ + else: + raise ValueError(f'Unknown distribution: {distribution}') + + section_bounds = [0, *torch.tensor(init_sections).cumsum(0).tolist()] + for i in range(len(init_sections)): + w = torch.empty((k, 1), dtype=R.dtype, device=R.device) + init_fn_(w) + R[:, section_bounds[i] : section_bounds[i + 1], 0] = w + # Note: do NOT scale by 1/sqrt(r) here — only path 0 contributes at init, + # so it must carry the full baseline magnitude. + + + + +def _mean_pairwise_jaccard(mask: np.ndarray) -> float: + if mask.shape[0] < 2: + return 1.0 + values: list[float] = [] + for i in range(mask.shape[0]): + for j in range(i + 1, mask.shape[0]): + union = np.logical_or(mask[i], mask[j]).sum() + values.append(1.0 if union == 0 else float(np.logical_and(mask[i], mask[j]).sum() / union)) + return float(np.mean(values)) if values else 1.0 + + +def _mfb_mask_coverage_stats(mask: np.ndarray) -> dict[str, float]: + features_per_member = mask.sum(axis=1) + members_per_feature = mask.sum(axis=0) + return { + 'keep_rate_actual': float(mask.mean()), + 'min_features_per_member': float(features_per_member.min()), + 'mean_features_per_member': float(features_per_member.mean()), + 'max_features_per_member': float(features_per_member.max()), + 'min_members_per_feature': float(members_per_feature.min()), + 'mean_members_per_feature': float(members_per_feature.mean()), + 'max_members_per_feature': float(members_per_feature.max()), + 'mean_pairwise_jaccard': _mean_pairwise_jaccard(mask), + } + + +def _make_mfb_feature_group_mask( + *, + k: int, + feature_widths: list[int], + keep_rate: float, + seed: int, + anchor_fraction: float = 0.0, + protected_feature_ids: None | list[int] = None, + ensure_each_feature_seen: bool = True, + ensure_each_member_nonempty: bool = True, +) -> tuple[np.ndarray, np.ndarray, dict[str, float]]: + feature_index: list[int] = [] + for feature_id, width in enumerate(feature_widths): + feature_index.extend([feature_id] * int(width)) + n_features = len(feature_widths) + gen = np.random.RandomState(seed) + feature_mask = (gen.rand(k, n_features) < keep_rate).astype(np.float32) + protected = np.array([] if protected_feature_ids is None else protected_feature_ids, dtype=np.int64) + n_anchor = max(0, min(k, int(round(k * anchor_fraction)))) + if protected.size > 0: + feature_mask[:, protected] = 1.0 + if n_anchor > 0: + feature_mask[:n_anchor, :] = 1.0 + if ensure_each_member_nonempty: + for row in range(k): + if feature_mask[row].sum() == 0: + feature_mask[row, gen.randint(0, n_features)] = 1.0 + if ensure_each_feature_seen: + for col in range(n_features): + if feature_mask[:, col].sum() == 0: + feature_mask[gen.randint(0, k), col] = 1.0 + dim_mask = feature_mask[:, np.asarray(feature_index, dtype=np.int64)] + stats = _mfb_mask_coverage_stats(feature_mask) + stats.update({ + 'anchor_fraction_configured': float(anchor_fraction), + 'n_anchor_members': float(n_anchor), + 'core_fraction_configured': float(len(protected) / max(1, n_features)), + 'n_core_features': float(len(protected)), + }) + return feature_mask.astype(np.float32), dim_mask.astype(np.float32), stats + + +def _get_first_adapter_for_cf_fisd(backbone: lib.deep.MLP) -> Tensor: + first = backbone.blocks[0][0] # type: ignore[code] + if isinstance(first, lib.deep.LinearEfficientEnsembleRankR): + return first.R[:, :, 0] + if isinstance(first, lib.deep.LinearEfficientEnsemble): + if first.r is None: + raise RuntimeError('CF-FISD requires first-layer input scaling.') + return first.r + raise RuntimeError(f'Unsupported first ensemble layer for CF-FISD: {type(first)!r}') + +DEFAULT_SHARE_TRAINING_BATCHES = True + + +class Model(nn.Module): + """MLP & TabM.""" + + def __init__( + self, + *, + n_num_features: int, + cat_cardinalities: list[int], + n_classes: None | int, + backbone: dict, + bins: None | list[Tensor], # For piecewise-linear encoding/embeddings. + num_embeddings: None | dict = None, + arch_type: Literal[ + # Plain feed-forward network without any kind of ensembling. + 'plain', + # + # TabM + 'tabm', + # + # TabM-mini + 'tabm-mini', + # + # TabM-packed + 'tabm-packed', + # + # TabM. The first adapter is initialized from the normal distribution. + # This variant was not used in the paper, but it may be useful in practice. + 'tabm-normal', + # + # TabM-mini. The adapter is initialized from the normal distribution. + # This variant was not used in the paper. + 'tabm-mini-normal', + ], + k: None | int = None, + share_training_batches: bool = DEFAULT_SHARE_TRAINING_BATCHES, + rla_rank: int = 1, + rla_first_only: bool = False, + rla_additive: bool = False, + rla_init: Literal['variance_preserving', 'base_preserving'] = 'variance_preserving', + rla_base_preserve_noise: float = 1e-3, + mfb: None | dict[str, Any] = None, + ) -> None: + # >>> Validate arguments. + assert n_num_features >= 0 + assert n_num_features or cat_cardinalities + if arch_type == 'plain': + assert k is None + assert ( + share_training_batches + ), 'If `arch_type` is set to "plain", then `simple` must remain True' + else: + assert k is not None + assert k > 0 + + super().__init__() + + # >>> Continuous (numerical) features + first_adapter_sections = [] # See the comment in `_init_first_adapter`. + + if n_num_features == 0: + assert bins is None + self.num_module = None + d_num = 0 + + elif num_embeddings is None: + assert bins is None + self.num_module = None + d_num = n_num_features + first_adapter_sections.extend(1 for _ in range(n_num_features)) + + else: + if bins is None: + self.num_module = lib.deep.make_module( + **num_embeddings, n_features=n_num_features + ) + else: + assert num_embeddings['type'].startswith('PiecewiseLinearEmbeddings') + self.num_module = lib.deep.make_module(**num_embeddings, bins=bins) + d_num = n_num_features * num_embeddings['d_embedding'] + first_adapter_sections.extend( + num_embeddings['d_embedding'] for _ in range(n_num_features) + ) + + # >>> Categorical features + self.cat_module = ( + lib.deep.OneHotEncoding0d(cat_cardinalities) if cat_cardinalities else None + ) + first_adapter_sections.extend(cat_cardinalities) + d_cat = sum(cat_cardinalities) + + # >>> Backbone + d_flat = d_num + d_cat + self.minimal_ensemble_adapter = None + backbone = dict(backbone) + backbone_type = backbone.pop('type', 'MLP') + self.backbone = lib.deep.make_module(backbone_type, d_in=d_flat, **backbone) + + if arch_type != 'plain': + assert k is not None + first_adapter_init = ( + None + if arch_type == 'tabm-packed' + else 'normal' + if arch_type in ('tabm-mini-normal', 'tabm-normal') + # For other arch_types, the initialization depends + # on the presense of num_embeddings. + else 'random-signs' + if num_embeddings is None + else 'normal' + ) + + if arch_type in ('tabm', 'tabm-normal'): + # Like BatchEnsemble, but all multiplicative adapters, + # except for the very first one, are initialized with ones. + assert first_adapter_init is not None + + use_rla = rla_rank > 1 or rla_additive + if not use_rla: + # Baseline path: rank-1 multiplicative adapter, identical + # to upstream TabM. This branch is kept exact so the rank-1 + # RLA exact-recovery test compares against the original. + lib.deep.make_efficient_ensemble( + self.backbone, + lib.deep.LinearEfficientEnsemble, + k=k, + ensemble_scaling_in=True, + ensemble_scaling_out=True, + ensemble_bias=True, + scaling_init='ones', + ) + _init_first_adapter( + _get_first_ensemble_layer(self.backbone).r, # type: ignore[code] + first_adapter_init, + first_adapter_sections, + ) + else: + # RLA path: rank-r (or additive) adapters. + if rla_first_only: + # All layers rank-1 baseline; replace only the first + # with the rank-r class. + lib.deep.make_efficient_ensemble( + self.backbone, + lib.deep.LinearEfficientEnsemble, + k=k, + ensemble_scaling_in=True, + ensemble_scaling_out=True, + ensemble_bias=True, + scaling_init='ones', + ) + _replace_first_with_rankr( + self.backbone, + k=k, + rank=rla_rank, + additive=rla_additive, + init_mode=rla_init, base_preserve_noise=rla_base_preserve_noise, + ) + else: + # RLA-uniform: every linear layer is rank-r / additive. + lib.deep.make_efficient_ensemble( + self.backbone, + lib.deep.LinearEfficientEnsembleRankR, + k=k, + rank=rla_rank, + scaling_init='ones', + additive=rla_additive, + init_mode=rla_init, base_preserve_noise=rla_base_preserve_noise, + ) + # First-layer R column initialisation: + # - variance_preserving: each rank column gets section-based + # random signs / normal, then scaled by 1/sqrt(rank). + # - base_preserving: path 0 already holds the proper + # section-based init from the layer constructor; extra + # paths stay zero/tiny so we *do not* overwrite them. + if not rla_additive and rla_init != 'base_preserving': + _init_first_adapter_rankr( + _get_first_rankr_layer(self.backbone).R, # type: ignore[code] + first_adapter_init, + first_adapter_sections, + ) + elif not rla_additive and rla_init == 'base_preserving': + # Apply the section-based init only to path 0 of R. + _init_first_adapter_basepreserve( + _get_first_rankr_layer(self.backbone).R, # type: ignore[code] + first_adapter_init, + first_adapter_sections, + ) + + elif arch_type in ('tabm-mini', 'tabm-mini-normal'): + # MiniEnsemble + assert first_adapter_init is not None + self.minimal_ensemble_adapter = lib.deep.ScaleEnsemble( + k, + d_flat, + init='random-signs' if num_embeddings is None else 'normal', + ) + _init_first_adapter( + self.minimal_ensemble_adapter.weight, # type: ignore[code] + first_adapter_init, + first_adapter_sections, + ) + + elif arch_type == 'tabm-packed': + # Packed ensemble. + # In terms of the Packed Ensembles paper by Laurent et al., + # TabM-packed is PackedEnsemble(alpha=k, M=k, gamma=1). + assert first_adapter_init is None + lib.deep.make_efficient_ensemble(self.backbone, lib.deep.NLinear, n=k) + + else: + raise ValueError(f'Unknown arch_type: {arch_type}') + + # >>> Output + d_block = backbone['d_block'] + d_out = 1 if n_classes is None else n_classes + self.output = ( + nn.Linear(d_block, d_out) + if arch_type == 'plain' + else lib.deep.NLinear(k, d_block, d_out) # type: ignore[code] + ) + + # >>> + self.arch_type = arch_type + self.k = k + self.share_training_batches = share_training_batches + self.mfb_cfg = {} if mfb is None else dict(mfb) + self.mfb_enabled = bool(self.mfb_cfg.get('enabled', False)) + self.mfb_mask_mode = str(self.mfb_cfg.get('mask_mode', 'member_fixed')) + self.mfb_mask_granularity = str(self.mfb_cfg.get('mask_granularity', 'feature_group')) + self.mfb_keep_rate = float(self.mfb_cfg.get('keep_rate', 1.0)) + self.mfb_inverted_scaling = bool(self.mfb_cfg.get('inverted_scaling', True)) + self.mfb_use_soft_mask = bool(self.mfb_cfg.get('use_soft_mask', False)) + self.mfb_mask_strength = float(self.mfb_cfg.get('mask_strength', 1.0)) + self.mfb_warmup_epochs = int(self.mfb_cfg.get('warmup_epochs', 0)) + self.mfb_start_epoch = int(self.mfb_cfg.get('start_epoch', 0)) + self.mfb_group_mode = str(self.mfb_cfg.get('group_mode', 'feature_group')) + self.mfb_categorical_handling = str(self.mfb_cfg.get('categorical_handling', 'drop_allowed')) + self.mfb_epoch = 0 + self.mfb_feature_widths = [int(x) for x in first_adapter_sections] + if self.mfb_enabled: + assert self.k is not None + if self.mfb_mask_granularity != 'feature_group': + raise ValueError(f'Unsupported MFB mask_granularity={self.mfb_mask_granularity!r}') + if self.mfb_group_mode not in {'feature_group', 'numerical_only', 'per_member'}: + raise ValueError(f'Unsupported MFB group_mode={self.mfb_group_mode!r}') + protected_feature_ids = list(self.mfb_cfg.get('protected_feature_ids') or []) + if ( + self.mfb_group_mode == 'numerical_only' + or self.mfb_categorical_handling in {'no_cat_drop', 'num_only'} + ): + protected_feature_ids.extend(range(n_num_features, n_num_features + len(cat_cardinalities))) + protected_feature_ids = sorted(set(int(x) for x in protected_feature_ids)) + feature_mask, dim_mask, mask_stats = _make_mfb_feature_group_mask( + k=self.k, + feature_widths=self.mfb_feature_widths, + keep_rate=self.mfb_keep_rate, + seed=int(self.mfb_cfg.get('mask_seed', 0)), + anchor_fraction=float(self.mfb_cfg.get('anchor_fraction', 0.0)), + protected_feature_ids=protected_feature_ids, + ) + self.register_buffer('mfb_fixed_feature_mask', torch.from_numpy(feature_mask), persistent=True) + self.register_buffer('mfb_fixed_dim_mask', torch.from_numpy(dim_mask), persistent=True) + self.mfb_mask_stats = mask_stats + else: + self.register_buffer('mfb_fixed_feature_mask', torch.empty((0, 0), dtype=torch.float32), persistent=True) + self.register_buffer('mfb_fixed_dim_mask', torch.empty((0, 0), dtype=torch.float32), persistent=True) + self.mfb_mask_stats = {'keep_rate_actual': 1.0} + + def set_epoch(self, epoch: int) -> None: + self.mfb_epoch = int(epoch) + + def _current_mfb_mask_strength(self) -> float: + if not self.mfb_use_soft_mask: + return 1.0 + if self.mfb_warmup_epochs <= 0: + return self.mfb_mask_strength + effective_epoch = max(0, self.mfb_epoch - self.mfb_start_epoch) + return self.mfb_mask_strength * min(1.0, max(0.0, float(effective_epoch) / float(self.mfb_warmup_epochs))) + + def _sample_mfb_mask(self, device_: torch.device, dtype: torch.dtype) -> Tensor: + assert self.k is not None + mask = (torch.rand((self.k, len(self.mfb_feature_widths)), device=device_) < self.mfb_keep_rate).to(dtype) + zero_rows = mask.sum(dim=1) == 0 + if bool(zero_rows.any()): + zero_indices = torch.where(zero_rows)[0] + random_cols = torch.randint(0, len(self.mfb_feature_widths), size=(len(zero_indices),), device=device_) + mask[zero_rows] = 0.0 + mask[zero_indices, random_cols] = 1.0 + feature_index = torch.tensor( + [feature_id for feature_id, width in enumerate(self.mfb_feature_widths) for _ in range(width)], + device=device_, + dtype=torch.long, + ) + return mask[:, feature_index] + + def _apply_mfb_mask(self, x: Tensor) -> Tensor: + if not self.mfb_enabled or self.mfb_mask_mode == 'none': + return x + if self.mfb_epoch < self.mfb_start_epoch: + return x + if self.mfb_mask_mode == 'member_fixed': + raw_mask = self.mfb_fixed_dim_mask.to(device=x.device, dtype=x.dtype) + elif self.mfb_mask_mode == 'stochastic': + if not self.training: + return x + raw_mask = self._sample_mfb_mask(x.device, x.dtype) + else: + raise ValueError(f'Unknown MFB mask_mode={self.mfb_mask_mode!r}') + if self.mfb_use_soft_mask and self.mfb_mask_mode == 'member_fixed': + alpha = self._current_mfb_mask_strength() + effective_mask = (1.0 - alpha) + alpha * raw_mask + return x * effective_mask.unsqueeze(0) + x = x * raw_mask.unsqueeze(0) + if self.mfb_inverted_scaling and self.mfb_keep_rate < 1.0: + x = x / max(self.mfb_keep_rate, 1e-6) + return x + + def forward( + self, x_num: None | Tensor = None, x_cat: None | Tensor = None + ) -> Tensor: + x = [] + if x_num is not None: + x.append(x_num if self.num_module is None else self.num_module(x_num)) + if x_cat is None: + assert self.cat_module is None + else: + assert self.cat_module is not None + x.append(self.cat_module(x_cat).float()) + x = torch.column_stack([x_.flatten(1, -1) for x_ in x]) + + if self.k is not None: + if self.share_training_batches or not self.training: + # (B, D) -> (B, K, D) + x = x[:, None].expand(-1, self.k, -1) + else: + # (B * K, D) -> (B, K, D) + x = x.reshape(len(x) // self.k, self.k, *x.shape[1:]) + if self.minimal_ensemble_adapter is not None: + x = self.minimal_ensemble_adapter(x) + x = self._apply_mfb_mask(x) + else: + assert self.minimal_ensemble_adapter is None + + x = self.backbone(x) + x = self.output(x) + if self.k is None: + # Adjust the output shape for plain networks to make them compatible + # with the rest of the script (loss, metrics, predictions, ...). + # (B, D_OUT) -> (B, 1, D_OUT) + x = x[:, None] + return x + + def get_member_logits( + self, x_num: None | Tensor = None, x_cat: None | Tensor = None + ) -> Tensor: + """Return per-member logits/predictions with shape (B, K, D_OUT).""" + return self.forward(x_num, x_cat) + + def get_first_batchensemble_r(self) -> None | Tensor: + """Expose the first BatchEnsemble `r` parameter when present.""" + if self.arch_type in ('tabm', 'tabm-normal'): + return _get_first_ensemble_layer(self.backbone).r + return None + + +class Config(TypedDict): + seed: int + data: KWArgs + bins: NotRequired[KWArgs] + model: KWArgs + head_selection: NotRequired[bool] + optimizer: KWArgs + n_lr_warmup_epochs: NotRequired[int] + batch_size: int + eval_batch_size: NotRequired[int] + patience: int + n_epochs: int + gradient_clipping_norm: NotRequired[float] + parameter_statistics: NotRequired[bool] + # NOTE + # Please, read these notes before using AMP and/or `torch.compile`. + # + # The usage of the following efficiency-related settings depends on the model. + # To learn if a given model can run with AMP and torch.compile on a given task, + # try activating these settings and check if the task metrics are satisfactory. + # The following notes can be helpful. + # + # - For simple architectures, such as MLP or TabM, these settings often + # make models significantly faster without any negative side-effects. + # For a real world task, it is worth to doublecheck that by comparing runs + # with and without AMP and/or torch.compile. + # + # - For more complex architectures, these settings should be used + # with extra caution. For example, some baselines used in this project showed + # worse performance when trained with AMP. For some models, AMP with BF16 hurts + # the performance, but AMP with FP16 works fine. Sometimes, it is the opposite. + # Sometimes, it depends on a dataset. Because of that, all baselines were run + # without AMP and torch.compile to ensure that results are representative. + # + # - AMP usually provides significantly larger speedups than `torch.compile`. + # So, if there are any issues with `torch.compile`, using only AMP will still + # lead to substantially faster models. + # + # - If a training run is already fast (e.g. on small datasets), + # `torch.compile` can make it *slower*, because the compilation itself + # takes some time (in particular, at the beginning of the first epoch, + # and at the beginning of the first evaluation). + # + # - Generally, compared to AMP, `torch.compile` is a younger technology, and a + # model must meet certain requirements to be compatible with `torch.compile`. + # In case of any issues, try updating PyTorch. + amp: NotRequired[bool] # torch.autocast + compile: NotRequired[bool] # torch.compile + use_ncl: NotRequired[bool] + lambda_ncl: NotRequired[float] + ncl_warmup_epochs: NotRequired[int] + ncl_space: NotRequired[Literal['logits', 'probs', 'hybrid']] + use_esam: NotRequired[bool] + esam_rho: NotRequired[float] + esam_eps: NotRequired[float] + esam_adapter_only: NotRequired[bool] + esam_memberwise: NotRequired[bool] + esam_warmup_epochs: NotRequired[int] + esam_start_epoch: NotRequired[int] + esam_end_epoch: NotRequired[int] + esam_log_diagnostics: NotRequired[bool] + esam_diagnostics_every: NotRequired[int] + rla_adapter_lr_multiplier: NotRequired[float] + rla_extra_paths_freeze_fraction: NotRequired[float] + cf_fisd: NotRequired[KWArgs] + + +def main( + config: Config | str | Path, + output: None | str | Path = None, + *, + force: bool = False, +) -> None | lib.JSONDict: + # >>> Start + config, output = lib.check(config, output, config_type=Config) + if not lib.start(output, force=force): + return None + + lib.print_config(config) # type: ignore[code] + delu.random.seed(config['seed']) + device = lib.get_device() + report = lib.create_report(main, config) + + # >>> Data + dataset = lib.data.build_dataset(**config['data']) + if dataset.task.is_regression: + dataset.data['y'], regression_label_stats = lib.data.standardize_labels( + dataset.data['y'] + ) + else: + regression_label_stats = None + + # Convert binary features to categorical features. + if dataset.n_bin_features > 0: + x_bin = dataset.data.pop('x_bin') + # Remove binary features with just one unique value in the training set. + # This must be done, otherwise, the script will fail on one specific dataset + # from the "why" benchmark. + n_bin_features = x_bin['train'].shape[1] + good_bin_idx = [ + i for i in range(n_bin_features) if len(np.unique(x_bin['train'][:, i])) > 1 + ] + if len(good_bin_idx) < n_bin_features: + x_bin = {k: v[:, good_bin_idx] for k, v in x_bin.items()} + + if dataset.n_cat_features == 0: + dataset.data['x_cat'] = { + part: np.zeros((dataset.size(part), 0), dtype=np.int64) + for part in x_bin + } + for part in x_bin: + dataset.data['x_cat'][part] = np.column_stack( + [dataset.data['x_cat'][part], x_bin[part].astype(np.int64)] + ) + del x_bin + dataset = dataset.to_torch(device) + Y_train = dataset.data['y']['train'].to( + torch.long if dataset.task.is_classification else torch.float + ) + + # >>> Model + if 'bins' in config: + # Compute the bins for PiecewiseLinearEncoding and PiecewiseLinearEmbeddings. + compute_bins_kwargs = ( + { + 'y': Y_train.to( + torch.long if dataset.task.is_classification else torch.float + ), + 'regression': dataset.task.is_regression, + 'verbose': True, + } + if 'tree_kwargs' in config['bins'] + else {} + ) + bin_edges = rtdl_num_embeddings.compute_bins( + dataset.data['x_num']['train'], **config['bins'], **compute_bins_kwargs + ) + logger.info(f'Bin counts: {[len(x) - 1 for x in bin_edges]}') + else: + bin_edges = None + model = Model( + n_num_features=dataset.n_num_features, + cat_cardinalities=dataset.compute_cat_cardinalities(), + n_classes=dataset.task.try_compute_n_classes(), + **config['model'], + bins=bin_edges, + ) + report['n_parameters'] = lib.deep.get_n_parameters(model) + logger.info(f'n_parameters = {report["n_parameters"]}') + report['prediction_type'] = 'labels' if dataset.task.is_regression else 'probs' + model.to(device) + if lib.is_dataparallel_available(): + model = nn.DataParallel(model) + + root_model = model.module if isinstance(model, nn.DataParallel) else model + report['mfb'] = { + 'enabled': bool(getattr(root_model, 'mfb_enabled', False)), + 'config': dict(getattr(root_model, 'mfb_cfg', {})), + 'mask_stats': dict(getattr(root_model, 'mfb_mask_stats', {})), + } + + cf_fisd_cfg = config.get('cf_fisd') + if cf_fisd_cfg: + cf_fisd_cat_cards = list(dataset.compute_cat_cardinalities()) + if 'num_embeddings' in config['model']: + cf_fisd_d_emb = int(config['model']['num_embeddings']['d_embedding']) + cf_fisd_d_features = [cf_fisd_d_emb] * dataset.n_num_features + cf_fisd_cat_cards + else: + cf_fisd_d_features = [1] * dataset.n_num_features + cf_fisd_cat_cards + cf_fisd_n_features = dataset.n_num_features + len(cf_fisd_cat_cards) + cf_fisd_teacher_names = tuple(cf_fisd_cfg.get('teacher_names', lib.cf_fisd.DEFAULT_TEACHER_NAMES)) + cf_fisd_teachers = lib.cf_fisd.load_teacher_importances( + cf_fisd_cfg['teacher_dir'], + cf_fisd_cfg['dataset_name'], + n_features=cf_fisd_n_features, + teacher_names=cf_fisd_teacher_names, + ) + cf_fisd_member_groups = ( + lib.cf_fisd.default_member_groups(int(config['model']['k']), cf_fisd_teacher_names) + if cf_fisd_cfg.get('member_groups') is None + else {k: list(v) for k, v in cf_fisd_cfg['member_groups'].items()} + ) + cf_fisd_lambda = float(cf_fisd_cfg.get('lambda', 0.0)) + cf_fisd_variant = str(cf_fisd_cfg.get('variant', 'raw')) + cf_fisd_start_epoch = int(cf_fisd_cfg.get('start_epoch', 0)) + cf_fisd_r1_param = _get_first_adapter_for_cf_fisd(root_model.backbone) + report['cf_fisd'] = { + 'lambda': cf_fisd_lambda, + 'variant': cf_fisd_variant, + 'teacher_names': list(cf_fisd_teacher_names), + 'member_groups': {k: list(v) for k, v in cf_fisd_member_groups.items()}, + 'd_features': list(cf_fisd_d_features), + 'n_features': cf_fisd_n_features, + 'teacher_dir': str(cf_fisd_cfg['teacher_dir']), + 'start_epoch': cf_fisd_start_epoch, + 'mode': cf_fisd_cfg.get('mode', cf_fisd_variant), + } + else: + cf_fisd_lambda = 0.0 + cf_fisd_variant = 'raw' + cf_fisd_d_features = [] + cf_fisd_teachers = {} + cf_fisd_member_groups = {} + cf_fisd_start_epoch = 0 + cf_fisd_r1_param = None + report['cf_fisd'] = {'lambda': 0.0} + + def compute_cf_fisd_penalty() -> Tensor: + if cf_fisd_r1_param is None or cf_fisd_lambda <= 0.0: + return Y_train.new_zeros((), dtype=torch.float32) + if step // epoch_size < cf_fisd_start_epoch: + return Y_train.new_zeros((), dtype=torch.float32) + return lib.cf_fisd.cf_fisd_loss( + cf_fisd_r1_param, + cf_fisd_teachers, + cf_fisd_member_groups, + cf_fisd_variant, + cf_fisd_d_features, + ) + + # >>> Training + step = 0 + batch_size = config['batch_size'] + report['epoch_size'] = epoch_size = math.ceil(dataset.size('train') / batch_size) + eval_batch_size = config.get( + 'eval_batch_size', + # With torch.compile, + # the largest possible evaluation batch size is noticeably smaller. + 2048 if config.get('compile', False) else 32768, + ) + chunk_size = None # Currently, not used. + share_training_batches = config['model'].get( + 'share_training_batches', DEFAULT_SHARE_TRAINING_BATCHES + ) + use_ncl = config.get('use_ncl', False) + lambda_ncl = float(config.get('lambda_ncl', 0.0)) + ncl_warmup_epochs = int(config.get('ncl_warmup_epochs', 0)) + ncl_space = config.get('ncl_space', 'logits') + if ncl_space not in ('logits', 'probs', 'hybrid'): + raise ValueError(f'Unknown ncl_space: {ncl_space}') + report['ncl'] = { + 'use_ncl': bool(use_ncl), + 'lambda_ncl': lambda_ncl, + 'ncl_warmup_epochs': ncl_warmup_epochs, + 'ncl_space': ncl_space, + } + use_esam = bool(config.get('use_esam', False)) + esam_rho = float(config.get('esam_rho', 0.01)) + esam_eps = float(config.get('esam_eps', 1e-12)) + esam_adapter_only = bool(config.get('esam_adapter_only', True)) + esam_memberwise = bool(config.get('esam_memberwise', True)) + esam_warmup_epochs = int(config.get('esam_warmup_epochs', 0)) + esam_start_epoch = int(config.get('esam_start_epoch', 0)) + esam_end_epoch = int(config.get('esam_end_epoch', -1)) + esam_log_diagnostics = bool(config.get('esam_log_diagnostics', True)) + esam_diagnostics_every = int(config.get('esam_diagnostics_every', 100)) + report['esam'] = { + 'use_esam': use_esam, + 'esam_rho': esam_rho, + 'esam_eps': esam_eps, + 'esam_adapter_only': esam_adapter_only, + 'esam_memberwise': esam_memberwise, + 'esam_warmup_epochs': esam_warmup_epochs, + 'esam_start_epoch': esam_start_epoch, + 'esam_end_epoch': esam_end_epoch, + 'esam_log_diagnostics': esam_log_diagnostics, + 'esam_diagnostics_every': esam_diagnostics_every, + } + + rla_adapter_lr_multiplier = float(config.get('rla_adapter_lr_multiplier', 1.0)) + rla_custom_groups = [] + if rla_adapter_lr_multiplier != 1.0: + rla_adapter_params = [ + p + for module in model.modules() + if isinstance(module, lib.deep.LinearEfficientEnsembleRankR) + for p in (module.R, module.S) + ] + if rla_adapter_params: + rla_custom_groups.append({ + 'params': rla_adapter_params, + 'lr': config['optimizer']['lr'] * rla_adapter_lr_multiplier, + }) + optimizer_cfg = dict(config['optimizer']) + optimizer_type = optimizer_cfg.pop('type', 'AdamW') + optimizer = lib.deep.make_optimizer( + optimizer_type, + **optimizer_cfg, + params=lib.deep.make_parameter_groups(model, custom_groups=rla_custom_groups), + ) + rla_extra_paths_freeze_fraction = float(config.get('rla_extra_paths_freeze_fraction', 0.0)) + assert 0.0 <= rla_extra_paths_freeze_fraction <= 1.0 + rla_extra_paths_freeze_until_epoch = ( + math.ceil(config['n_epochs'] * rla_extra_paths_freeze_fraction) + if config['n_epochs'] > 0 + else math.ceil(2 * config['patience'] * rla_extra_paths_freeze_fraction) + ) + report['rla_training'] = { + 'adapter_lr_multiplier': rla_adapter_lr_multiplier, + 'extra_paths_freeze_fraction': rla_extra_paths_freeze_fraction, + 'extra_paths_freeze_until_epoch': rla_extra_paths_freeze_until_epoch, + } + + def zero_rla_extra_path_grads() -> None: + if rla_extra_paths_freeze_fraction == 0.0: + return + if step // epoch_size >= rla_extra_paths_freeze_until_epoch: + return + for module in model.modules(): + if isinstance(module, lib.deep.LinearEfficientEnsembleRankR) and module.rank > 1: + if module.R.grad is not None: + module.R.grad[:, :, 1:].zero_() + if module.S.grad is not None: + module.S.grad[:, :, 1:].zero_() + gradient_clipping_norm = config.get('gradient_clipping_norm') + _loss_fn = ( + nn.functional.mse_loss + if dataset.task.is_regression + else nn.functional.cross_entropy + ) + + def loss_fn(y_pred: Tensor, y_true: Tensor) -> Tensor: + return _loss_fn( + y_pred.flatten(0, 1), + ( + y_true.repeat_interleave(y_pred.shape[1]) + if share_training_batches + else y_true + ), + ) + + def ncl_penalty(y_pred: Tensor) -> Tensor: + if y_pred.shape[1] <= 1: + return y_pred.new_zeros(()) + + def corr_penalty(member_outputs: Tensor) -> Tensor: + if member_outputs.ndim == 2: + member_outputs = member_outputs.unsqueeze(-1) + centered = member_outputs - member_outputs.mean(dim=1, keepdim=True) + norms = centered.norm(dim=-1) + denom = torch.einsum('bi,bj->bij', norms, norms).clamp_min(1e-8) + corr = torch.einsum('bid,bjd->bij', centered, centered) / denom + k_members = centered.shape[1] + off_diag = ~torch.eye( + k_members, dtype=torch.bool, device=centered.device + ).unsqueeze(0) + return corr.masked_select(off_diag).mean() + + if not dataset.task.is_classification: + return corr_penalty(y_pred) + + if ncl_space == 'logits': + return corr_penalty(y_pred) + if ncl_space == 'probs': + return corr_penalty(y_pred.softmax(-1)) + # Hybrid penalty: equal weighting between logits-space and probability-space. + return 0.5 * (corr_penalty(y_pred) + corr_penalty(y_pred.softmax(-1))) + + base_model = model.module if isinstance(model, nn.DataParallel) else model + + def get_esam_adapter_params() -> list[tuple[str, nn.Parameter]]: + params: list[tuple[str, nn.Parameter]] = [] + if not esam_adapter_only: + params.extend( + [ + (name, param) + for name, param in base_model.named_parameters() + if param.requires_grad + ] + ) + return params + + for module_name, module in base_model.named_modules(): + if isinstance(module, lib.deep.LinearEfficientEnsembleRankR): + for pname in ['R', 'S', 'bias']: + p = getattr(module, pname) + if p is None or not p.requires_grad: + continue + params.append((f'{module_name}.{pname}', p)) + elif isinstance(module, lib.deep.LinearEfficientEnsemble): + for pname in ['r', 's', 'bias']: + p = getattr(module, pname) + if p is None or not p.requires_grad: + continue + if pname == 'bias' and p.ndim != 2: + # Shared bias is not member-specific. + continue + params.append((f'{module_name}.{pname}', p)) + elif isinstance(module, lib.deep.ScaleEnsemble): + if module.weight.requires_grad: + params.append((f'{module_name}.weight', module.weight)) + + if not params: + logger.warning('ESAM adapter-only selection found no adapter parameters.') + return params + + esam_adapter_named_params = get_esam_adapter_params() + esam_adapter_params = [p for _, p in esam_adapter_named_params] + report['esam']['adapter_parameter_names'] = [ + {'name': name, 'shape': list(param.shape)} + for name, param in esam_adapter_named_params + ] + logger.info( + 'ESAM adapter params: {}', + [f'{name}:{tuple(param.shape)}' for name, param in esam_adapter_named_params], + ) + + def compute_esam_perturbations( + grads: list[None | Tensor], + rho: float, + eps: float, + memberwise: bool, + ) -> tuple[list[Tensor], float, float, bool]: + if not grads: + return [], 0.0, 0.0, True + + grad_norm_sq = 0.0 + fallback_used = False + for param, grad in zip(esam_adapter_params, grads): + if grad is None: + continue + if memberwise and param.ndim >= 2 and param.shape[0] > 1: + member_norms = grad.reshape(param.shape[0], -1).norm(dim=1) + grad_norm_sq += float((member_norms**2).sum().item()) + else: + if memberwise: + fallback_used = True + grad_norm_sq += float((grad.norm() ** 2).item()) + grad_norm = math.sqrt(grad_norm_sq) + + perturbations: list[Tensor] = [] + perturb_norm_sq = 0.0 + for param, grad in zip(esam_adapter_params, grads): + if grad is None: + perturb = torch.zeros_like(param) + elif memberwise and param.ndim >= 2 and param.shape[0] > 1: + member_norms = grad.reshape(param.shape[0], -1).norm(dim=1) + view_shape = (param.shape[0],) + (1,) * (param.ndim - 1) + scale = rho / (member_norms.view(view_shape) + eps) + perturb = grad * scale + else: + perturb = grad * (rho / (grad_norm + eps)) + perturbations.append(perturb) + perturb_norm_sq += float((perturb.norm() ** 2).item()) + perturb_norm = math.sqrt(perturb_norm_sq) + return perturbations, grad_norm, perturb_norm, fallback_used + + def apply_esam_perturbation(perturbations: list[Tensor]) -> None: + with torch.no_grad(): + for param, perturb in zip(esam_adapter_params, perturbations): + param.add_(perturb) + + def restore_esam_perturbation(perturbations: list[Tensor]) -> float: + max_abs_restore = 0.0 + with torch.no_grad(): + for param, perturb in zip(esam_adapter_params, perturbations): + param.sub_(perturb) + if perturb.numel(): + max_abs_restore = max(max_abs_restore, float(perturb.abs().max().item())) + return max_abs_restore + + # The following generator is used only for creating training batches, + # so the random seed fully determines the sequence of training objects. + batch_generator = torch.Generator(device).manual_seed(config['seed']) + timer = delu.tools.Timer() + early_stopping = delu.tools.EarlyStopping(config['patience'], mode='max') + parameter_statistics = config.get('parameter_statistics', config['seed'] == 1) + training_log = [] + esam_diag_records: list[dict[str, Any]] = [] + writer = torch.utils.tensorboard.SummaryWriter(output) # type: ignore[code] + writer_failed = False + run_id = f'{output.parent.name}__{output.name}' + esam_diag_path = (output.parent / '_esam_diagnostics') + esam_diag_file = esam_diag_path / f'{run_id}.jsonl' + if use_esam and esam_log_diagnostics: + esam_diag_path.mkdir(parents=True, exist_ok=True) + esam_diag_file.write_text('') + + def safe_add_scalars(tag: str, values: dict[str, Any], step_: int, walltime: float) -> None: + nonlocal writer, writer_failed + if writer is None or writer_failed: + return + try: + writer.add_scalars(tag, values, step_, walltime) + except OSError as err: + writer_failed = True + logger.warning(f'TensorBoard write disabled due to OS error: {err}') + try: + writer.close() + except Exception: + pass + writer = None + + # Only bfloat16 was tested as amp_dtype. + # However, float16 is supported as a fallback. + # To enable float16, uncomment the two lines below. + amp_dtype = ( + torch.bfloat16 + if config.get('amp', False) + and torch.cuda.is_available() + and torch.cuda.is_bf16_supported() + # else torch.float16 + # if config.get('amp', False) and and torch.cuda.is_available() + else None + ) + amp_enabled = amp_dtype is not None + # For FP16, the gradient scaler must be used. + grad_scaler = torch.cuda.amp.GradScaler() if amp_dtype is torch.float16 else None # type: ignore[code] + logger.info(f'AMP enabled: {amp_enabled}') + + if config.get('compile', False): + # NOTE + # `torch.compile` is intentionally called without the `mode` argument, + # because it caused issues with training. + model = torch.compile(model) + evaluation_mode = torch.no_grad + else: + evaluation_mode = torch.inference_mode + + @torch.autocast(device.type, enabled=amp_enabled, dtype=amp_dtype) # type: ignore[code] + def apply_model(part: PartKey, idx: Tensor) -> Tensor: + return ( + model( + dataset.data['x_num'][part][idx] if 'x_num' in dataset.data else None, + dataset.data['x_cat'][part][idx] if 'x_cat' in dataset.data else None, + ) + .squeeze(-1) # Remove the last dimension for regression predictions. + .float() + ) + + @evaluation_mode() + def evaluate( + parts: list[PartKey], eval_batch_size: int + ) -> tuple[ + dict[PartKey, Any], + dict[PartKey, np.ndarray], + dict[PartKey, np.ndarray], + dict[PartKey, Any], + int, + ]: + def compute_diversity(head_preds: np.ndarray) -> dict[str, float]: + if head_preds.ndim == 2: + members = head_preds[..., None] + else: + members = head_preds + members = members.reshape(members.shape[0], members.shape[1], -1) + if members.shape[1] <= 1: + return { + 'mean_centered_corr': 0.0, + 'mean_pairwise_disagreement': 0.0, + 'member_std': 0.0, + } + + centered = members - members.mean(axis=1, keepdims=True) + norms = np.linalg.norm(centered, axis=-1) + denom = np.einsum('bi,bj->bij', norms, norms) + 1e-8 + corr = np.einsum('bid,bjd->bij', centered, centered) / denom + off_diag = ~np.eye(members.shape[1], dtype=bool) + mean_centered_corr = float(corr[:, off_diag].mean()) + + if dataset.task.is_regression: + pairwise_disagreement = float('nan') + elif dataset.task.is_binclass: + labels = (head_preds > 0.5).astype(np.int64) + pairwise_disagreement = float( + (labels[:, :, None] != labels[:, None, :])[:, off_diag].mean() + ) + else: + labels = head_preds.argmax(-1) + pairwise_disagreement = float( + (labels[:, :, None] != labels[:, None, :])[:, off_diag].mean() + ) + + member_std = float(members.std(axis=1).mean()) + return { + 'mean_centered_corr': mean_centered_corr, + 'mean_pairwise_disagreement': pairwise_disagreement, + 'member_std': member_std, + } + + model.eval() + head_predictions: dict[PartKey, np.ndarray] = {} + for part in parts: + while eval_batch_size: + try: + head_predictions[part] = ( + torch.cat( + [ + apply_model(part, idx) + for idx in torch.arange( + dataset.size(part), device=device + ).split(eval_batch_size) + ] + ) + .cpu() + .numpy() + ) + except RuntimeError as err: + if not lib.is_oom_exception(err): + raise + eval_batch_size //= 2 + logger.warning(f'eval_batch_size = {eval_batch_size}') + else: + break + if not eval_batch_size: + RuntimeError('Not enough memory even for eval_batch_size=1') + if dataset.task.is_regression: + assert regression_label_stats is not None + head_predictions = { + k: v * regression_label_stats.std + regression_label_stats.mean + for k, v in head_predictions.items() + } + else: + head_predictions = { + k: scipy.special.softmax(v, axis=-1) + for k, v in head_predictions.items() + } + if dataset.task.is_binclass: + head_predictions = {k: v[..., 1] for k, v in head_predictions.items()} + + predictions = {k: v.mean(1) for k, v in head_predictions.items()} + metrics = ( + dataset.task.calculate_metrics(predictions, report['prediction_type']) + if lib.are_valid_predictions(predictions) + else {x: {'score': lib.WORST_SCORE} for x in predictions} + ) + diversity = {part: compute_diversity(head_predictions[part]) for part in parts} + return metrics, predictions, head_predictions, diversity, eval_batch_size + + def save_checkpoint() -> None: + lib.dump_checkpoint( + output, + { + 'step': step, + 'model': model.state_dict(), + 'optimizer': optimizer.state_dict(), + 'batch_generator': batch_generator.get_state(), + 'random_state': delu.random.get_state(), + 'early_stopping': early_stopping, + 'report': report, + 'timer': timer, + 'training_log': training_log, + } + | ( + {} if grad_scaler is None else {'grad_scaler': grad_scaler.state_dict()} + ), + ) + lib.dump_report(output, report) + lib.backup_output(output) + + print() + timer.run() + while config['n_epochs'] == -1 or step // epoch_size < config['n_epochs']: + print(f'[...] {lib.try_get_relative_path(output)} | {timer}') + + model.train() + if hasattr(model, 'set_epoch'): + model.set_epoch(step // epoch_size + 1) + epoch_total_losses = [] + epoch_task_losses = [] + epoch_ncl_losses = [] + epoch_esam_sharpness = [] + epoch_esam_grad_norm = [] + epoch_esam_perturb_norm = [] + batches = ( + torch.randperm( + dataset.size('train'), + generator=batch_generator, + device=device, + ).split(batch_size) + if share_training_batches + else [ + x.transpose(0, 1).flatten() + for x in torch.rand( + (config['model']['k'], dataset.size('train')), + generator=batch_generator, + device=device, + ) + .argsort(dim=1) + .split(batch_size, dim=1) + ] + ) + for batch_idx in tqdm(batches, desc=f'Epoch {step // epoch_size} Step {step}'): + optimizer.zero_grad() + y_pred = apply_model('train', batch_idx) + task_loss = loss_fn(y_pred, Y_train[batch_idx]) + ncl_raw = ( + ncl_penalty(y_pred) + if use_ncl and lambda_ncl > 0.0 and y_pred.shape[1] > 1 + else task_loss.new_zeros(()) + ) + if use_ncl and lambda_ncl > 0.0 and ncl_warmup_epochs > 0: + warmup_factor = min( + 1.0, float(step // epoch_size + 1) / float(ncl_warmup_epochs) + ) + else: + warmup_factor = 1.0 + lambda_effective = lambda_ncl * warmup_factor if use_ncl else 0.0 + cf_fisd_penalty = compute_cf_fisd_penalty() + clean_loss = task_loss + lambda_effective * ncl_raw + cf_fisd_lambda * cf_fisd_penalty + if not torch.isfinite(clean_loss): + report['failure'] = {'reason': 'non_finite_loss', 'step': int(step), 'stage': 'clean'} + lib.dump_report(output, report) + raise RuntimeError(f'Non-finite clean loss at step {step}') + + epoch = step // epoch_size + esam_enabled_now = ( + use_esam + and esam_rho > 0.0 + and bool(esam_adapter_params) + and epoch >= esam_start_epoch + and (esam_end_epoch < 0 or epoch <= esam_end_epoch) + ) + if esam_enabled_now and esam_warmup_epochs > 0: + esam_rho_effective = esam_rho * min( + 1.0, float(epoch - esam_start_epoch + 1) / float(esam_warmup_epochs) + ) + else: + esam_rho_effective = esam_rho + + esam_clean = float(clean_loss.detach().item()) + esam_perturbed = esam_clean + esam_sharpness = 0.0 + esam_grad_norm = 0.0 + esam_perturb_norm = 0.0 + esam_fallback_used = False + esam_restore_max_abs = 0.0 + esam_pending_perturbations = None + loss = clean_loss + + if esam_enabled_now and esam_rho_effective > 0.0: + grads = torch.autograd.grad( + clean_loss, + esam_adapter_params, + retain_graph=False, + create_graph=False, + allow_unused=True, + ) + perturbations, esam_grad_norm, esam_perturb_norm, esam_fallback_used = ( + compute_esam_perturbations( + list(grads), + esam_rho_effective, + esam_eps, + esam_memberwise, + ) + ) + apply_esam_perturbation(perturbations) + esam_pending_perturbations = perturbations + try: + y_pred_perturbed = apply_model('train', batch_idx) + pert_task_loss = loss_fn(y_pred_perturbed, Y_train[batch_idx]) + pert_ncl_raw = ( + ncl_penalty(y_pred_perturbed) + if use_ncl and lambda_ncl > 0.0 and y_pred_perturbed.shape[1] > 1 + else pert_task_loss.new_zeros(()) + ) + pert_cf_fisd_penalty = compute_cf_fisd_penalty() + pert_loss = pert_task_loss + lambda_effective * pert_ncl_raw + cf_fisd_lambda * pert_cf_fisd_penalty + if not torch.isfinite(pert_loss): + report['failure'] = {'reason': 'non_finite_loss', 'step': int(step), 'stage': 'perturbed'} + lib.dump_report(output, report) + raise RuntimeError(f'Non-finite perturbed loss at step {step}') + except Exception: + esam_restore_max_abs = restore_esam_perturbation(perturbations) + raise + loss = pert_loss + esam_perturbed = float(pert_loss.detach().item()) + esam_sharpness = esam_perturbed - esam_clean + + optimizer.zero_grad() + if grad_scaler is None: + loss.backward() + else: + grad_scaler.scale(loss).backward() + + if esam_pending_perturbations is not None: + esam_restore_max_abs = restore_esam_perturbation(esam_pending_perturbations) + + zero_rla_extra_path_grads() + + if parameter_statistics and ( + step % epoch_size == 0 # The first batch of the epoch. + or step // epoch_size == 0 # The first epoch. + ): + for k, v in lib.deep.compute_parameter_stats(model).items(): + safe_add_scalars(k, v, step, timer.elapsed()) + del k, v + + if gradient_clipping_norm is not None: + if grad_scaler is not None: + grad_scaler.unscale_(optimizer) + nn.utils.clip_grad.clip_grad_norm_( + model.parameters(), gradient_clipping_norm + ) + if grad_scaler is None: + optimizer.step() + else: + grad_scaler.step(optimizer) + grad_scaler.update() + + step += 1 + epoch_total_losses.append(loss.detach()) + epoch_task_losses.append(task_loss.detach()) + epoch_ncl_losses.append(ncl_raw.detach()) + if esam_enabled_now: + epoch_esam_sharpness.append(esam_sharpness) + epoch_esam_grad_norm.append(esam_grad_norm) + epoch_esam_perturb_norm.append(esam_perturb_norm) + + if use_esam and esam_log_diagnostics and (step % max(1, esam_diagnostics_every) == 0): + rec = { + 'step': step, + 'epoch': epoch, + 'dataset': str(config['data']['path']), + 'esam_enabled_now': esam_enabled_now, + 'esam_fallback_mode': bool(esam_fallback_used), + 'esam_rho_effective': float(esam_rho_effective), + 'clean_loss': esam_clean, + 'perturbed_loss': esam_perturbed, + 'sharpness_proxy': esam_sharpness, + 'adapter_grad_norm': esam_grad_norm, + 'adapter_perturbation_norm': esam_perturb_norm, + 'restore_max_abs_perturb': esam_restore_max_abs, + 'member_loss_available': bool(y_pred.shape[1] > 1), + # Only populated in a follow-up pass where labels are aligned per member. + 'train_member_loss_mean': None, + } + esam_diag_records.append(rec) + with esam_diag_file.open('a') as f: + f.write(json.dumps(rec) + '\n') + + epoch_total_losses = torch.stack(epoch_total_losses).tolist() + epoch_task_losses = torch.stack(epoch_task_losses).tolist() + epoch_ncl_losses = torch.stack(epoch_ncl_losses).tolist() + mean_esam_sharpness = ( + statistics.mean(epoch_esam_sharpness) if epoch_esam_sharpness else 0.0 + ) + mean_esam_grad_norm = ( + statistics.mean(epoch_esam_grad_norm) if epoch_esam_grad_norm else 0.0 + ) + mean_esam_perturb_norm = ( + statistics.mean(epoch_esam_perturb_norm) if epoch_esam_perturb_norm else 0.0 + ) + mean_loss = statistics.mean(epoch_total_losses) + mean_task_loss = statistics.mean(epoch_task_losses) + mean_ncl_loss = statistics.mean(epoch_ncl_losses) + metrics, predictions, _, diversity, eval_batch_size = evaluate( + ['val', 'test'], eval_batch_size + ) + + training_log.append( + { + 'epoch-losses': epoch_total_losses, + 'epoch-task-losses': epoch_task_losses, + 'epoch-ncl-losses': epoch_ncl_losses, + 'lambda_ncl_effective': lambda_effective, + 'esam_mean_sharpness': mean_esam_sharpness, + 'esam_mean_grad_norm': mean_esam_grad_norm, + 'esam_mean_perturb_norm': mean_esam_perturb_norm, + 'metrics': metrics, + 'diversity': diversity, + 'time': timer.elapsed(), + } + ) + lib.print_metrics(mean_loss, metrics) + safe_add_scalars( + 'loss', + { + 'train_total': mean_loss, + 'train_task': mean_task_loss, + 'train_ncl': mean_ncl_loss, + 'train_esam_sharpness': mean_esam_sharpness, + 'train_esam_grad_norm': mean_esam_grad_norm, + 'train_esam_perturb_norm': mean_esam_perturb_norm, + }, + step, + timer.elapsed(), + ) + for part in metrics: + safe_add_scalars( + 'score', {part: metrics[part]['score']}, step, timer.elapsed() + ) + for part in diversity: + safe_add_scalars( + 'diversity/mean_centered_corr', + {part: diversity[part]['mean_centered_corr']}, + step, + timer.elapsed(), + ) + safe_add_scalars( + 'diversity/member_std', + {part: diversity[part]['member_std']}, + step, + timer.elapsed(), + ) + if not math.isnan(diversity[part]['mean_pairwise_disagreement']): + safe_add_scalars( + 'diversity/mean_pairwise_disagreement', + {part: diversity[part]['mean_pairwise_disagreement']}, + step, + timer.elapsed(), + ) + + if ( + 'metrics' not in report + or metrics['val']['score'] > report['metrics']['val']['score'] + ): + print('🌸 New best epoch! 🌸') + report['best_step'] = step + report['metrics'] = metrics + save_checkpoint() + lib.dump_predictions(output, predictions) + + early_stopping.update(metrics['val']['score']) + if early_stopping.should_stop() or not lib.are_valid_predictions(predictions): + break + + print() + report['time'] = str(timer) + + # >>> + if lib.get_checkpoint_path(output).exists(): + model.load_state_dict(lib.load_checkpoint(output)['model']) + report['metrics'], predictions, head_predictions, report['diversity'], _ = evaluate( + ['train', 'val', 'test'], eval_batch_size + ) + report['chunk_size'] = chunk_size + report['eval_batch_size'] = eval_batch_size + report['esam']['diagnostics_path'] = str(esam_diag_file) if use_esam else None + report['esam']['n_diagnostics_records'] = len(esam_diag_records) + lib.dump_predictions(output, predictions) + lib.dump_summary(output, lib.summarize(report)) + save_checkpoint() + + # >>> Submodel selection (TabM[B] & TabM[G]). + if ( + config.get('head_selection', True) + and head_predictions['train'].shape[1] > 1 + # The following conditions is a hack preventing the head selection during + # the hyperparameter tuning, because bin/tune.py runs training + # outside of the project directory. + and lib.env.get_project_dir() in output.parents + and output.parent.name != 'trials' + ): + if output.parent.name.endswith('-evaluation'): + best_head_output = ( + output.parent.with_name( + output.parent.name.removesuffix('-evaluation') + + '-best-head-evaluation' + ) + / output.name + ) + greedy_heads_output = ( + output.parent.with_name( + output.parent.name.removesuffix('-evaluation') + + '-greedy-heads-evaluation' + ) + / output.name + ) + else: + best_head_output = output.with_name(output.name + '-best-head') + greedy_heads_output = output.with_name(output.name + '-greedy-heads') + for dir_ in [best_head_output, greedy_heads_output]: + if dir_.exists(): + logger.warning(f'Removing the existing output: {dir_}') + shutil.rmtree(dir_) + + prediction_type = ( + lib.PredictionType.PROBS + if dataset.task.is_classification + else lib.PredictionType.LABELS + ) + head_selection_timer = delu.tools.Timer() + head_selection_timer.run() + + # >>> TabM[B]: select the Best submodel. + n_heads = head_predictions['val'].shape[1] + head_val_scores = np.array( + [ + dataset.task.calculate_metrics( + {'val': head_predictions['val'][:, i]}, prediction_type + )['val']['score'] + for i in range(n_heads) + ] + ) + best_head_idx = int(np.argmax(head_val_scores)) + best_head_output.mkdir(parents=True) + lib.finish( + best_head_output, + report + | { + 'heads': [best_head_idx], + 'head_selection_time': str(head_selection_timer), + 'metrics': dataset.task.calculate_metrics( + {k: v[:, best_head_idx] for k, v in head_predictions.items()}, + prediction_type, + ), + }, + ) + + # >>> TabM[G]: Greedily select a powerful subset of submodels. + + # Start with the best head. + greedy_idx = [best_head_idx] + greedy_score = head_val_scores[best_head_idx] + + greedy_mask = [False] * n_heads + greedy_mask[best_head_idx] = True + + while len(greedy_idx) < n_heads: + new_idx = None + new_score = None + + # Iterating through all heads. + for head_idx in range(n_heads): + # If the head is already in greedy_idx, skip it. + if greedy_mask[head_idx]: + continue + + candidate_idx = [*greedy_idx, head_idx] + candidate_score = dataset.task.calculate_metrics( + {'val': head_predictions['val'][:, candidate_idx].mean(1)}, + prediction_type, + )['val']['score'] + if candidate_score > greedy_score and ( + new_score is None or candidate_score > new_score + ): + new_idx = candidate_idx + new_score = candidate_score + + # If no head improves the current greedy score, + # the head selection process is stopped. + if new_idx is None: + break + else: + assert new_score is not None + greedy_score = new_score + greedy_idx = new_idx + + greedy_heads_output.mkdir(parents=True) + lib.finish( + greedy_heads_output, + report + | { + 'heads': greedy_idx, + 'head_selection_time': str(head_selection_timer), + 'metrics': dataset.task.calculate_metrics( + {k: v[:, greedy_idx].mean(1) for k, v in head_predictions.items()}, + prediction_type, + ), + }, + ) + + lib.finish(output, report) + return report + + +if __name__ == '__main__': + lib.configure_libraries() + lib.run(main) diff --git a/tabm_integrated/paper/data/.gitignore b/tabm_integrated/paper/data/.gitignore new file mode 100644 index 000000000..d6b7ef32c --- /dev/null +++ b/tabm_integrated/paper/data/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/tabm_integrated/paper/exp/.gitignore b/tabm_integrated/paper/exp/.gitignore new file mode 100644 index 000000000..f87762f76 --- /dev/null +++ b/tabm_integrated/paper/exp/.gitignore @@ -0,0 +1,17 @@ +_logs/ +_queue_*.txt +_queue_*.txt.lock +_queue_*.txt.tmp +*.out +FAILED +*.pt +*.pth +checkpoint* +checkpoint.pt +checkpoint_best.pt +predictions*.npz +summary.json +events.out.tfevents.* +cache/ +__pycache__/ +_esam_diagnostics/ diff --git a/tabm_integrated/paper/exp/final_integrated_audit.csv b/tabm_integrated/paper/exp/final_integrated_audit.csv new file mode 100644 index 000000000..b9f49e6cc --- /dev/null +++ b/tabm_integrated/paper/exp/final_integrated_audit.csv @@ -0,0 +1,1588 @@ +wave,dataset,variant,seed,metric,metric_key,direction,validation_metric,validation_score,test_metric,inference_mode,config_path,result_path,failure,best_step +baseline_fidelity,sberbank-housing,baseline_plr,0,RMSE,rmse,lower,0.23452539387100468,-0.23452539387100468,0.23846659721529556,mean,exp/integrated/baseline_fidelity/sberbank-housing/baseline_plr-evaluation/0.toml,exp/integrated/baseline_fidelity/sberbank-housing/baseline_plr-evaluation/0,False,814 +baseline_fidelity,sberbank-housing,baseline_plr,0,RMSE,rmse,lower,0.23955454428788905,-0.23955454428788905,0.26588264760298186,best-head,exp/integrated/baseline_fidelity/sberbank-housing/baseline_plr-evaluation/0.toml,exp/integrated/baseline_fidelity/sberbank-housing/baseline_plr-best-head-evaluation/0,False,814 +baseline_fidelity,sberbank-housing,baseline_plr,0,RMSE,rmse,lower,0.23257941028161666,-0.23257941028161666,0.23549883338311248,greedy-heads,exp/integrated/baseline_fidelity/sberbank-housing/baseline_plr-evaluation/0.toml,exp/integrated/baseline_fidelity/sberbank-housing/baseline_plr-greedy-heads-evaluation/0,False,814 +baseline_fidelity,ecom-offers,baseline_plr,0,AUROC,roc-auc,higher,0.6227054517511603,0.6227054517511603,0.591151981311678,mean,exp/integrated/baseline_fidelity/ecom-offers/baseline_plr-evaluation/0.toml,exp/integrated/baseline_fidelity/ecom-offers/baseline_plr-evaluation/0,False,321 +baseline_fidelity,ecom-offers,baseline_plr,0,AUROC,roc-auc,higher,0.6404461738890127,0.6404461738890127,0.6034781946000056,best-head,exp/integrated/baseline_fidelity/ecom-offers/baseline_plr-evaluation/0.toml,exp/integrated/baseline_fidelity/ecom-offers/baseline_plr-best-head-evaluation/0,False,321 +baseline_fidelity,ecom-offers,baseline_plr,0,AUROC,roc-auc,higher,0.6413971749638401,0.6413971749638401,0.6040305589800323,greedy-heads,exp/integrated/baseline_fidelity/ecom-offers/baseline_plr-evaluation/0.toml,exp/integrated/baseline_fidelity/ecom-offers/baseline_plr-greedy-heads-evaluation/0,False,321 +baseline_fidelity,homesite-insurance,baseline_plr,0,AUROC,roc-auc,higher,0.9583121450834431,0.9583121450834431,0.9626260391126089,mean,exp/integrated/baseline_fidelity/homesite-insurance/baseline_plr-evaluation/0.toml,exp/integrated/baseline_fidelity/homesite-insurance/baseline_plr-evaluation/0,False,1980 +baseline_fidelity,homesite-insurance,baseline_plr,0,AUROC,roc-auc,higher,0.9554763470552776,0.9554763470552776,0.959416932798502,best-head,exp/integrated/baseline_fidelity/homesite-insurance/baseline_plr-evaluation/0.toml,exp/integrated/baseline_fidelity/homesite-insurance/baseline_plr-best-head-evaluation/0,False,1980 +baseline_fidelity,homesite-insurance,baseline_plr,0,AUROC,roc-auc,higher,0.9589474980445032,0.9589474980445032,0.9625170670540112,greedy-heads,exp/integrated/baseline_fidelity/homesite-insurance/baseline_plr-evaluation/0.toml,exp/integrated/baseline_fidelity/homesite-insurance/baseline_plr-greedy-heads-evaluation/0,False,1980 +baseline_fidelity,cooking-time,baseline_plr,0,RMSE,rmse,lower,0.4627596029926704,-0.4627596029926704,0.4802386655626689,mean,exp/integrated/baseline_fidelity/cooking-time/baseline_plr-evaluation/0.toml,exp/integrated/baseline_fidelity/cooking-time/baseline_plr-evaluation/0,False,23088 +baseline_fidelity,cooking-time,baseline_plr,0,RMSE,rmse,lower,0.4652274815891859,-0.4652274815891859,0.4820660657381116,best-head,exp/integrated/baseline_fidelity/cooking-time/baseline_plr-evaluation/0.toml,exp/integrated/baseline_fidelity/cooking-time/baseline_plr-best-head-evaluation/0,False,23088 +baseline_fidelity,cooking-time,baseline_plr,0,RMSE,rmse,lower,0.4625169879783321,-0.4625169879783321,0.47990786370986477,greedy-heads,exp/integrated/baseline_fidelity/cooking-time/baseline_plr-evaluation/0.toml,exp/integrated/baseline_fidelity/cooking-time/baseline_plr-greedy-heads-evaluation/0,False,23088 +baseline_fidelity,delivery-eta,baseline_plr,0,RMSE,rmse,lower,0.5558749906446844,-0.5558749906446844,0.5496162918403876,mean,exp/integrated/baseline_fidelity/delivery-eta/baseline_plr-evaluation/0.toml,exp/integrated/baseline_fidelity/delivery-eta/baseline_plr-evaluation/0,False,2184 +baseline_fidelity,delivery-eta,baseline_plr,0,RMSE,rmse,lower,0.5583881453279098,-0.5583881453279098,0.5557654209580059,best-head,exp/integrated/baseline_fidelity/delivery-eta/baseline_plr-evaluation/0.toml,exp/integrated/baseline_fidelity/delivery-eta/baseline_plr-best-head-evaluation/0,False,2184 +baseline_fidelity,delivery-eta,baseline_plr,0,RMSE,rmse,lower,0.5553348954063525,-0.5553348954063525,0.550360417400049,greedy-heads,exp/integrated/baseline_fidelity/delivery-eta/baseline_plr-evaluation/0.toml,exp/integrated/baseline_fidelity/delivery-eta/baseline_plr-greedy-heads-evaluation/0,False,2184 +smoke,sberbank-housing,baseline_plr,0,RMSE,rmse,lower,0.23452539387100468,-0.23452539387100468,0.23846659721529556,mean,exp/integrated/smoke/sberbank-housing/baseline_plr-evaluation/0.toml,exp/integrated/smoke/sberbank-housing/baseline_plr-evaluation/0,False,814 +smoke,sberbank-housing,rla_only,0,RMSE,rmse,lower,0.2354418316334552,-0.2354418316334552,0.23162903716515967,mean,exp/integrated/smoke/sberbank-housing/rla_only-evaluation/0.toml,exp/integrated/smoke/sberbank-housing/rla_only-evaluation/0,False,666 +smoke,sberbank-housing,esam_only,0,RMSE,rmse,lower,0.23533073111501152,-0.23533073111501152,0.24162357572341994,mean,exp/integrated/smoke/sberbank-housing/esam_only-evaluation/0.toml,exp/integrated/smoke/sberbank-housing/esam_only-evaluation/0,False,888 +smoke,sberbank-housing,mfb_only,0,RMSE,rmse,lower,0.23520949733542967,-0.23520949733542967,0.23880983570262945,mean,exp/integrated/smoke/sberbank-housing/mfb_only-evaluation/0.toml,exp/integrated/smoke/sberbank-housing/mfb_only-evaluation/0,False,1184 +smoke,sberbank-housing,cf_fisd_only,0,RMSE,rmse,lower,0.23523413233394516,-0.23523413233394516,0.235149605552611,mean,exp/integrated/smoke/sberbank-housing/cf_fisd_only-evaluation/0.toml,exp/integrated/smoke/sberbank-housing/cf_fisd_only-evaluation/0,False,666 +smoke,sberbank-housing,all_four_combined,0,RMSE,rmse,lower,0.2348792849855834,-0.2348792849855834,0.2326272249658597,mean,exp/integrated/smoke/sberbank-housing/all_four_combined-evaluation/0.toml,exp/integrated/smoke/sberbank-housing/all_four_combined-evaluation/0,False,666 +smoke,sberbank-housing,baseline_plr,0,RMSE,rmse,lower,0.23955454428788905,-0.23955454428788905,0.26588264760298186,best-head,exp/integrated/smoke/sberbank-housing/baseline_plr-evaluation/0.toml,exp/integrated/smoke/sberbank-housing/baseline_plr-best-head-evaluation/0,False,814 +smoke,sberbank-housing,baseline_plr,0,RMSE,rmse,lower,0.23257941028161666,-0.23257941028161666,0.23549883338311248,greedy-heads,exp/integrated/smoke/sberbank-housing/baseline_plr-evaluation/0.toml,exp/integrated/smoke/sberbank-housing/baseline_plr-greedy-heads-evaluation/0,False,814 +smoke,sberbank-housing,cf_fisd_only,0,RMSE,rmse,lower,0.2388197411165918,-0.2388197411165918,0.2462523116085554,best-head,exp/integrated/smoke/sberbank-housing/cf_fisd_only-evaluation/0.toml,exp/integrated/smoke/sberbank-housing/cf_fisd_only-best-head-evaluation/0,False,666 +smoke,sberbank-housing,cf_fisd_only,0,RMSE,rmse,lower,0.23327114512588593,-0.23327114512588593,0.23367141178994547,greedy-heads,exp/integrated/smoke/sberbank-housing/cf_fisd_only-evaluation/0.toml,exp/integrated/smoke/sberbank-housing/cf_fisd_only-greedy-heads-evaluation/0,False,666 +smoke,sberbank-housing,mfb_only,0,RMSE,rmse,lower,0.24356476580119216,-0.24356476580119216,0.2459548830871388,best-head,exp/integrated/smoke/sberbank-housing/mfb_only-evaluation/0.toml,exp/integrated/smoke/sberbank-housing/mfb_only-best-head-evaluation/0,False,1184 +smoke,sberbank-housing,mfb_only,0,RMSE,rmse,lower,0.233840053737128,-0.233840053737128,0.23548819505307594,greedy-heads,exp/integrated/smoke/sberbank-housing/mfb_only-evaluation/0.toml,exp/integrated/smoke/sberbank-housing/mfb_only-greedy-heads-evaluation/0,False,1184 +smoke,sberbank-housing,esam_only,0,RMSE,rmse,lower,0.2398028724080995,-0.2398028724080995,0.23711371881317633,best-head,exp/integrated/smoke/sberbank-housing/esam_only-evaluation/0.toml,exp/integrated/smoke/sberbank-housing/esam_only-best-head-evaluation/0,False,888 +smoke,sberbank-housing,esam_only,0,RMSE,rmse,lower,0.23380275644783471,-0.23380275644783471,0.24257603366460162,greedy-heads,exp/integrated/smoke/sberbank-housing/esam_only-evaluation/0.toml,exp/integrated/smoke/sberbank-housing/esam_only-greedy-heads-evaluation/0,False,888 +smoke,sberbank-housing,rla_only,0,RMSE,rmse,lower,0.23765212789668771,-0.23765212789668771,0.24778267915496135,best-head,exp/integrated/smoke/sberbank-housing/rla_only-evaluation/0.toml,exp/integrated/smoke/sberbank-housing/rla_only-best-head-evaluation/0,False,666 +smoke,sberbank-housing,rla_only,0,RMSE,rmse,lower,0.2336025622310011,-0.2336025622310011,0.23389241285301454,greedy-heads,exp/integrated/smoke/sberbank-housing/rla_only-evaluation/0.toml,exp/integrated/smoke/sberbank-housing/rla_only-greedy-heads-evaluation/0,False,666 +smoke,sberbank-housing,all_four_combined,0,RMSE,rmse,lower,0.2376718937442043,-0.2376718937442043,0.2539783962159738,best-head,exp/integrated/smoke/sberbank-housing/all_four_combined-evaluation/0.toml,exp/integrated/smoke/sberbank-housing/all_four_combined-best-head-evaluation/0,False,666 +smoke,sberbank-housing,all_four_combined,0,RMSE,rmse,lower,0.23335850745168515,-0.23335850745168515,0.23263890687901315,greedy-heads,exp/integrated/smoke/sberbank-housing/all_four_combined-evaluation/0.toml,exp/integrated/smoke/sberbank-housing/all_four_combined-greedy-heads-evaluation/0,False,666 +smoke,ecom-offers,baseline_plr,0,AUROC,roc-auc,higher,0.6227054517511603,0.6227054517511603,0.591151981311678,mean,exp/integrated/smoke/ecom-offers/baseline_plr-evaluation/0.toml,exp/integrated/smoke/ecom-offers/baseline_plr-evaluation/0,False,321 +smoke,ecom-offers,rla_only,0,AUROC,roc-auc,higher,0.6226365549199251,0.6226365549199251,0.5911320299643263,mean,exp/integrated/smoke/ecom-offers/rla_only-evaluation/0.toml,exp/integrated/smoke/ecom-offers/rla_only-evaluation/0,False,214 +smoke,ecom-offers,esam_only,0,AUROC,roc-auc,higher,0.6227207494642631,0.6227207494642631,0.5911612394559284,mean,exp/integrated/smoke/ecom-offers/esam_only-evaluation/0.toml,exp/integrated/smoke/ecom-offers/esam_only-evaluation/0,False,321 +smoke,ecom-offers,mfb_only,0,AUROC,roc-auc,higher,0.6250090898119413,0.6250090898119413,0.5927112610884333,mean,exp/integrated/smoke/ecom-offers/mfb_only-evaluation/0.toml,exp/integrated/smoke/ecom-offers/mfb_only-evaluation/0,False,321 +smoke,ecom-offers,cf_fisd_only,0,AUROC,roc-auc,higher,0.6228570653182646,0.6228570653182646,0.5912083608375949,mean,exp/integrated/smoke/ecom-offers/cf_fisd_only-evaluation/0.toml,exp/integrated/smoke/ecom-offers/cf_fisd_only-evaluation/0,False,321 +smoke,ecom-offers,all_four_combined,0,AUROC,roc-auc,higher,0.6244437709503304,0.6244437709503304,0.592244305956134,mean,exp/integrated/smoke/ecom-offers/all_four_combined-evaluation/0.toml,exp/integrated/smoke/ecom-offers/all_four_combined-evaluation/0,False,321 +smoke,ecom-offers,baseline_plr,0,AUROC,roc-auc,higher,0.6404461738890127,0.6404461738890127,0.6034781946000056,best-head,exp/integrated/smoke/ecom-offers/baseline_plr-evaluation/0.toml,exp/integrated/smoke/ecom-offers/baseline_plr-best-head-evaluation/0,False,321 +smoke,ecom-offers,mfb_only,0,AUROC,roc-auc,higher,0.6501781765294031,0.6501781765294031,0.6048261425892079,best-head,exp/integrated/smoke/ecom-offers/mfb_only-evaluation/0.toml,exp/integrated/smoke/ecom-offers/mfb_only-best-head-evaluation/0,False,321 +smoke,ecom-offers,cf_fisd_only,0,AUROC,roc-auc,higher,0.640920668337369,0.640920668337369,0.6022998594088651,best-head,exp/integrated/smoke/ecom-offers/cf_fisd_only-evaluation/0.toml,exp/integrated/smoke/ecom-offers/cf_fisd_only-best-head-evaluation/0,False,321 +smoke,ecom-offers,mfb_only,0,AUROC,roc-auc,higher,0.6506721330887921,0.6506721330887921,0.6064667434006787,greedy-heads,exp/integrated/smoke/ecom-offers/mfb_only-evaluation/0.toml,exp/integrated/smoke/ecom-offers/mfb_only-greedy-heads-evaluation/0,False,321 +smoke,ecom-offers,cf_fisd_only,0,AUROC,roc-auc,higher,0.6418041361448852,0.6418041361448852,0.6039617691395597,greedy-heads,exp/integrated/smoke/ecom-offers/cf_fisd_only-evaluation/0.toml,exp/integrated/smoke/ecom-offers/cf_fisd_only-greedy-heads-evaluation/0,False,321 +smoke,ecom-offers,esam_only,0,AUROC,roc-auc,higher,0.6406711951042247,0.6406711951042247,0.6023965414622969,best-head,exp/integrated/smoke/ecom-offers/esam_only-evaluation/0.toml,exp/integrated/smoke/ecom-offers/esam_only-best-head-evaluation/0,False,321 +smoke,ecom-offers,esam_only,0,AUROC,roc-auc,higher,0.6415752785241469,0.6415752785241469,0.6040898334194843,greedy-heads,exp/integrated/smoke/ecom-offers/esam_only-evaluation/0.toml,exp/integrated/smoke/ecom-offers/esam_only-greedy-heads-evaluation/0,False,321 +smoke,ecom-offers,baseline_plr,0,AUROC,roc-auc,higher,0.6413971749638401,0.6413971749638401,0.6040305589800323,greedy-heads,exp/integrated/smoke/ecom-offers/baseline_plr-evaluation/0.toml,exp/integrated/smoke/ecom-offers/baseline_plr-greedy-heads-evaluation/0,False,321 +smoke,ecom-offers,rla_only,0,AUROC,roc-auc,higher,0.6394530055211662,0.6394530055211662,0.6014026252811646,best-head,exp/integrated/smoke/ecom-offers/rla_only-evaluation/0.toml,exp/integrated/smoke/ecom-offers/rla_only-best-head-evaluation/0,False,214 +smoke,ecom-offers,rla_only,0,AUROC,roc-auc,higher,0.6394530055211662,0.6394530055211662,0.6014026252811646,greedy-heads,exp/integrated/smoke/ecom-offers/rla_only-evaluation/0.toml,exp/integrated/smoke/ecom-offers/rla_only-greedy-heads-evaluation/0,False,214 +smoke,ecom-offers,all_four_combined,0,AUROC,roc-auc,higher,0.6479868261151002,0.6479868261151002,0.6047175337399694,best-head,exp/integrated/smoke/ecom-offers/all_four_combined-evaluation/0.toml,exp/integrated/smoke/ecom-offers/all_four_combined-best-head-evaluation/0,False,321 +smoke,ecom-offers,all_four_combined,0,AUROC,roc-auc,higher,0.6505096994576862,0.6505096994576862,0.6063775930816054,greedy-heads,exp/integrated/smoke/ecom-offers/all_four_combined-evaluation/0.toml,exp/integrated/smoke/ecom-offers/all_four_combined-greedy-heads-evaluation/0,False,321 +smoke,homesite-insurance,baseline_plr,0,AUROC,roc-auc,higher,0.9583121450834431,0.9583121450834431,0.9626260391126089,mean,exp/integrated/smoke/homesite-insurance/baseline_plr-evaluation/0.toml,exp/integrated/smoke/homesite-insurance/baseline_plr-evaluation/0,False,1980 +smoke,homesite-insurance,rla_only,0,AUROC,roc-auc,higher,0.9583231528038466,0.9583231528038466,0.96242593886798,mean,exp/integrated/smoke/homesite-insurance/rla_only-evaluation/0.toml,exp/integrated/smoke/homesite-insurance/rla_only-evaluation/0,False,1980 +smoke,homesite-insurance,esam_only,0,AUROC,roc-auc,higher,0.9583120395781356,0.9583120395781356,0.9627483366901237,mean,exp/integrated/smoke/homesite-insurance/esam_only-evaluation/0.toml,exp/integrated/smoke/homesite-insurance/esam_only-evaluation/0,False,1980 +smoke,homesite-insurance,mfb_only,0,AUROC,roc-auc,higher,0.9580942942079151,0.9580942942079151,0.9616652728230258,mean,exp/integrated/smoke/homesite-insurance/mfb_only-evaluation/0.toml,exp/integrated/smoke/homesite-insurance/mfb_only-evaluation/0,False,1980 +smoke,homesite-insurance,cf_fisd_only,0,AUROC,roc-auc,higher,0.9589599740471012,0.9589599740471012,0.9628908822909436,mean,exp/integrated/smoke/homesite-insurance/cf_fisd_only-evaluation/0.toml,exp/integrated/smoke/homesite-insurance/cf_fisd_only-evaluation/0,False,2420 +smoke,homesite-insurance,all_four_combined,0,AUROC,roc-auc,higher,0.958474253988237,0.958474253988237,0.962487069534998,mean,exp/integrated/smoke/homesite-insurance/all_four_combined-evaluation/0.toml,exp/integrated/smoke/homesite-insurance/all_four_combined-evaluation/0,False,1760 +smoke,homesite-insurance,baseline_plr,0,AUROC,roc-auc,higher,0.9554763470552776,0.9554763470552776,0.959416932798502,best-head,exp/integrated/smoke/homesite-insurance/baseline_plr-evaluation/0.toml,exp/integrated/smoke/homesite-insurance/baseline_plr-best-head-evaluation/0,False,1980 +smoke,homesite-insurance,baseline_plr,0,AUROC,roc-auc,higher,0.9589474980445032,0.9589474980445032,0.9625170670540112,greedy-heads,exp/integrated/smoke/homesite-insurance/baseline_plr-evaluation/0.toml,exp/integrated/smoke/homesite-insurance/baseline_plr-greedy-heads-evaluation/0,False,1980 +smoke,homesite-insurance,mfb_only,0,AUROC,roc-auc,higher,0.9560981250001208,0.9560981250001208,0.9601610718172741,best-head,exp/integrated/smoke/homesite-insurance/mfb_only-evaluation/0.toml,exp/integrated/smoke/homesite-insurance/mfb_only-best-head-evaluation/0,False,1980 +smoke,homesite-insurance,mfb_only,0,AUROC,roc-auc,higher,0.9587921238951639,0.9587921238951639,0.9622146031450255,greedy-heads,exp/integrated/smoke/homesite-insurance/mfb_only-evaluation/0.toml,exp/integrated/smoke/homesite-insurance/mfb_only-greedy-heads-evaluation/0,False,1980 +smoke,homesite-insurance,cf_fisd_only,0,AUROC,roc-auc,higher,0.957813544584969,0.957813544584969,0.961490313471295,best-head,exp/integrated/smoke/homesite-insurance/cf_fisd_only-evaluation/0.toml,exp/integrated/smoke/homesite-insurance/cf_fisd_only-best-head-evaluation/0,False,2420 +smoke,homesite-insurance,cf_fisd_only,0,AUROC,roc-auc,higher,0.9594844321379927,0.9594844321379927,0.9634382071404677,greedy-heads,exp/integrated/smoke/homesite-insurance/cf_fisd_only-evaluation/0.toml,exp/integrated/smoke/homesite-insurance/cf_fisd_only-greedy-heads-evaluation/0,False,2420 +smoke,homesite-insurance,esam_only,0,AUROC,roc-auc,higher,0.9561512908829329,0.9561512908829329,0.9606910602029306,best-head,exp/integrated/smoke/homesite-insurance/esam_only-evaluation/0.toml,exp/integrated/smoke/homesite-insurance/esam_only-best-head-evaluation/0,False,1980 +smoke,homesite-insurance,esam_only,0,AUROC,roc-auc,higher,0.958991133281215,0.958991133281215,0.9627938464179217,greedy-heads,exp/integrated/smoke/homesite-insurance/esam_only-evaluation/0.toml,exp/integrated/smoke/homesite-insurance/esam_only-greedy-heads-evaluation/0,False,1980 +smoke,homesite-insurance,rla_only,0,AUROC,roc-auc,higher,0.9557610707114682,0.9557610707114682,0.9604946447119459,best-head,exp/integrated/smoke/homesite-insurance/rla_only-evaluation/0.toml,exp/integrated/smoke/homesite-insurance/rla_only-best-head-evaluation/0,False,1980 +smoke,homesite-insurance,rla_only,0,AUROC,roc-auc,higher,0.959029757015826,0.959029757015826,0.9624579176971494,greedy-heads,exp/integrated/smoke/homesite-insurance/rla_only-evaluation/0.toml,exp/integrated/smoke/homesite-insurance/rla_only-greedy-heads-evaluation/0,False,1980 +smoke,homesite-insurance,all_four_combined,0,AUROC,roc-auc,higher,0.9569685349939213,0.9569685349939213,0.9582203301945192,best-head,exp/integrated/smoke/homesite-insurance/all_four_combined-evaluation/0.toml,exp/integrated/smoke/homesite-insurance/all_four_combined-best-head-evaluation/0,False,1760 +smoke,homesite-insurance,all_four_combined,0,AUROC,roc-auc,higher,0.9592737644154757,0.9592737644154757,0.9627526496640615,greedy-heads,exp/integrated/smoke/homesite-insurance/all_four_combined-evaluation/0.toml,exp/integrated/smoke/homesite-insurance/all_four_combined-greedy-heads-evaluation/0,False,1760 +smoke,cooking-time,baseline_plr,0,RMSE,rmse,lower,0.4627596029926704,-0.4627596029926704,0.4802386655626689,mean,exp/integrated/smoke/cooking-time/baseline_plr-evaluation/0.toml,exp/integrated/smoke/cooking-time/baseline_plr-evaluation/0,False,23088 +smoke,cooking-time,rla_only,0,RMSE,rmse,lower,0.4627297681263241,-0.4627297681263241,0.48025858554781287,mean,exp/integrated/smoke/cooking-time/rla_only-evaluation/0.toml,exp/integrated/smoke/cooking-time/rla_only-evaluation/0,False,23088 +smoke,cooking-time,esam_only,0,RMSE,rmse,lower,0.4628077726618535,-0.4628077726618535,0.48022789849772834,mean,exp/integrated/smoke/cooking-time/esam_only-evaluation/0.toml,exp/integrated/smoke/cooking-time/esam_only-evaluation/0,False,23088 +smoke,cooking-time,mfb_only,0,RMSE,rmse,lower,0.46290981075910853,-0.46290981075910853,0.480415480847424,mean,exp/integrated/smoke/cooking-time/mfb_only-evaluation/0.toml,exp/integrated/smoke/cooking-time/mfb_only-evaluation/0,False,23088 +smoke,cooking-time,cf_fisd_only,0,RMSE,rmse,lower,0.4632809116304394,-0.4632809116304394,0.48091704691326154,mean,exp/integrated/smoke/cooking-time/cf_fisd_only-evaluation/0.toml,exp/integrated/smoke/cooking-time/cf_fisd_only-evaluation/0,False,17982 +smoke,cooking-time,all_four_combined,0,RMSE,rmse,lower,0.463355012524859,-0.463355012524859,0.4809202538385541,mean,exp/integrated/smoke/cooking-time/all_four_combined-evaluation/0.toml,exp/integrated/smoke/cooking-time/all_four_combined-evaluation/0,False,17982 +smoke,cooking-time,baseline_plr,0,RMSE,rmse,lower,0.4652274815891859,-0.4652274815891859,0.4820660657381116,best-head,exp/integrated/smoke/cooking-time/baseline_plr-evaluation/0.toml,exp/integrated/smoke/cooking-time/baseline_plr-best-head-evaluation/0,False,23088 +smoke,cooking-time,baseline_plr,0,RMSE,rmse,lower,0.4625169879783321,-0.4625169879783321,0.47990786370986477,greedy-heads,exp/integrated/smoke/cooking-time/baseline_plr-evaluation/0.toml,exp/integrated/smoke/cooking-time/baseline_plr-greedy-heads-evaluation/0,False,23088 +smoke,cooking-time,cf_fisd_only,0,RMSE,rmse,lower,0.46397138529545745,-0.46397138529545745,0.4813020969034895,best-head,exp/integrated/smoke/cooking-time/cf_fisd_only-evaluation/0.toml,exp/integrated/smoke/cooking-time/cf_fisd_only-best-head-evaluation/0,False,17982 +smoke,cooking-time,cf_fisd_only,0,RMSE,rmse,lower,0.4629530240908241,-0.4629530240908241,0.480628211999089,greedy-heads,exp/integrated/smoke/cooking-time/cf_fisd_only-evaluation/0.toml,exp/integrated/smoke/cooking-time/cf_fisd_only-greedy-heads-evaluation/0,False,17982 +smoke,cooking-time,mfb_only,0,RMSE,rmse,lower,0.465403548325863,-0.465403548325863,0.48311284423774803,best-head,exp/integrated/smoke/cooking-time/mfb_only-evaluation/0.toml,exp/integrated/smoke/cooking-time/mfb_only-best-head-evaluation/0,False,23088 +smoke,cooking-time,mfb_only,0,RMSE,rmse,lower,0.4625593198933149,-0.4625593198933149,0.479899744054174,greedy-heads,exp/integrated/smoke/cooking-time/mfb_only-evaluation/0.toml,exp/integrated/smoke/cooking-time/mfb_only-greedy-heads-evaluation/0,False,23088 +smoke,cooking-time,esam_only,0,RMSE,rmse,lower,0.4651283707110853,-0.4651283707110853,0.4823068025931435,best-head,exp/integrated/smoke/cooking-time/esam_only-evaluation/0.toml,exp/integrated/smoke/cooking-time/esam_only-best-head-evaluation/0,False,23088 +smoke,cooking-time,esam_only,0,RMSE,rmse,lower,0.4625398779765968,-0.4625398779765968,0.4797024392153251,greedy-heads,exp/integrated/smoke/cooking-time/esam_only-evaluation/0.toml,exp/integrated/smoke/cooking-time/esam_only-greedy-heads-evaluation/0,False,23088 +smoke,cooking-time,rla_only,0,RMSE,rmse,lower,0.4651183431272534,-0.4651183431272534,0.48213882482447107,best-head,exp/integrated/smoke/cooking-time/rla_only-evaluation/0.toml,exp/integrated/smoke/cooking-time/rla_only-best-head-evaluation/0,False,23088 +smoke,cooking-time,rla_only,0,RMSE,rmse,lower,0.4624375326912171,-0.4624375326912171,0.48004674599435365,greedy-heads,exp/integrated/smoke/cooking-time/rla_only-evaluation/0.toml,exp/integrated/smoke/cooking-time/rla_only-greedy-heads-evaluation/0,False,23088 +smoke,cooking-time,all_four_combined,0,RMSE,rmse,lower,0.46391938568359065,-0.46391938568359065,0.48142889277254225,best-head,exp/integrated/smoke/cooking-time/all_four_combined-evaluation/0.toml,exp/integrated/smoke/cooking-time/all_four_combined-best-head-evaluation/0,False,17982 +smoke,cooking-time,all_four_combined,0,RMSE,rmse,lower,0.4629352081332738,-0.4629352081332738,0.4802514336856013,greedy-heads,exp/integrated/smoke/cooking-time/all_four_combined-evaluation/0.toml,exp/integrated/smoke/cooking-time/all_four_combined-greedy-heads-evaluation/0,False,17982 +smoke,delivery-eta,baseline_plr,0,RMSE,rmse,lower,0.5558749906446844,-0.5558749906446844,0.5496162918403876,mean,exp/integrated/smoke/delivery-eta/baseline_plr-evaluation/0.toml,exp/integrated/smoke/delivery-eta/baseline_plr-evaluation/0,False,2184 +smoke,delivery-eta,rla_only,0,RMSE,rmse,lower,0.5556926485294449,-0.5556926485294449,0.5502675684403805,mean,exp/integrated/smoke/delivery-eta/rla_only-evaluation/0.toml,exp/integrated/smoke/delivery-eta/rla_only-evaluation/0,False,1365 +smoke,delivery-eta,esam_only,0,RMSE,rmse,lower,0.5556360918188752,-0.5556360918188752,0.5494649592968895,mean,exp/integrated/smoke/delivery-eta/esam_only-evaluation/0.toml,exp/integrated/smoke/delivery-eta/esam_only-evaluation/0,False,2184 +smoke,delivery-eta,mfb_only,0,RMSE,rmse,lower,0.5560026829864069,-0.5560026829864069,0.5501201530906976,mean,exp/integrated/smoke/delivery-eta/mfb_only-evaluation/0.toml,exp/integrated/smoke/delivery-eta/mfb_only-evaluation/0,False,2184 +smoke,delivery-eta,cf_fisd_only,0,RMSE,rmse,lower,0.5562691235201772,-0.5562691235201772,0.5515996385557942,mean,exp/integrated/smoke/delivery-eta/cf_fisd_only-evaluation/0.toml,exp/integrated/smoke/delivery-eta/cf_fisd_only-evaluation/0,False,3003 +smoke,delivery-eta,all_four_combined,0,RMSE,rmse,lower,0.5560789252392828,-0.5560789252392828,0.5493995976501785,mean,exp/integrated/smoke/delivery-eta/all_four_combined-evaluation/0.toml,exp/integrated/smoke/delivery-eta/all_four_combined-evaluation/0,False,1638 +smoke,delivery-eta,baseline_plr,0,RMSE,rmse,lower,0.5583881453279098,-0.5583881453279098,0.5557654209580059,best-head,exp/integrated/smoke/delivery-eta/baseline_plr-evaluation/0.toml,exp/integrated/smoke/delivery-eta/baseline_plr-best-head-evaluation/0,False,2184 +smoke,delivery-eta,baseline_plr,0,RMSE,rmse,lower,0.5553348954063525,-0.5553348954063525,0.550360417400049,greedy-heads,exp/integrated/smoke/delivery-eta/baseline_plr-evaluation/0.toml,exp/integrated/smoke/delivery-eta/baseline_plr-greedy-heads-evaluation/0,False,2184 +smoke,delivery-eta,mfb_only,0,RMSE,rmse,lower,0.5590040394051904,-0.5590040394051904,0.5557457406296488,best-head,exp/integrated/smoke/delivery-eta/mfb_only-evaluation/0.toml,exp/integrated/smoke/delivery-eta/mfb_only-best-head-evaluation/0,False,2184 +smoke,delivery-eta,mfb_only,0,RMSE,rmse,lower,0.5555016895512279,-0.5555016895512279,0.5503428181880856,greedy-heads,exp/integrated/smoke/delivery-eta/mfb_only-evaluation/0.toml,exp/integrated/smoke/delivery-eta/mfb_only-greedy-heads-evaluation/0,False,2184 +smoke,delivery-eta,cf_fisd_only,0,RMSE,rmse,lower,0.5580564462352832,-0.5580564462352832,0.5528824139554394,best-head,exp/integrated/smoke/delivery-eta/cf_fisd_only-evaluation/0.toml,exp/integrated/smoke/delivery-eta/cf_fisd_only-best-head-evaluation/0,False,3003 +smoke,delivery-eta,cf_fisd_only,0,RMSE,rmse,lower,0.5559326756199947,-0.5559326756199947,0.5522105562224877,greedy-heads,exp/integrated/smoke/delivery-eta/cf_fisd_only-evaluation/0.toml,exp/integrated/smoke/delivery-eta/cf_fisd_only-greedy-heads-evaluation/0,False,3003 +smoke,delivery-eta,esam_only,0,RMSE,rmse,lower,0.5582863288518123,-0.5582863288518123,0.5560390501031047,best-head,exp/integrated/smoke/delivery-eta/esam_only-evaluation/0.toml,exp/integrated/smoke/delivery-eta/esam_only-best-head-evaluation/0,False,2184 +smoke,delivery-eta,esam_only,0,RMSE,rmse,lower,0.5551513290724005,-0.5551513290724005,0.5507488105906536,greedy-heads,exp/integrated/smoke/delivery-eta/esam_only-evaluation/0.toml,exp/integrated/smoke/delivery-eta/esam_only-greedy-heads-evaluation/0,False,2184 +smoke,delivery-eta,rla_only,0,RMSE,rmse,lower,0.5573103773832353,-0.5573103773832353,0.5532468973961396,best-head,exp/integrated/smoke/delivery-eta/rla_only-evaluation/0.toml,exp/integrated/smoke/delivery-eta/rla_only-best-head-evaluation/0,False,1365 +smoke,delivery-eta,rla_only,0,RMSE,rmse,lower,0.5553051102575359,-0.5553051102575359,0.5508335169889497,greedy-heads,exp/integrated/smoke/delivery-eta/rla_only-evaluation/0.toml,exp/integrated/smoke/delivery-eta/rla_only-greedy-heads-evaluation/0,False,1365 +smoke,delivery-eta,all_four_combined,0,RMSE,rmse,lower,0.557107642501001,-0.557107642501001,0.5520572896128711,best-head,exp/integrated/smoke/delivery-eta/all_four_combined-evaluation/0.toml,exp/integrated/smoke/delivery-eta/all_four_combined-best-head-evaluation/0,False,1638 +smoke,delivery-eta,all_four_combined,0,RMSE,rmse,lower,0.5558210261849236,-0.5558210261849236,0.551614874495521,greedy-heads,exp/integrated/smoke/delivery-eta/all_four_combined-evaluation/0.toml,exp/integrated/smoke/delivery-eta/all_four_combined-greedy-heads-evaluation/0,False,1638 +sweeps,sberbank-housing,rla_rank2_noise0.001,0,RMSE,rmse,lower,0.2354682854519358,-0.2354682854519358,0.24014251007009368,mean,exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.001-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.001-evaluation/0,False,814 +sweeps,sberbank-housing,rla_rank2_noise0.0001,0,RMSE,rmse,lower,0.23622468781228076,-0.23622468781228076,0.23935440604862293,mean,exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.0001-evaluation/0,False,962 +sweeps,sberbank-housing,rla_rank2_noise1e-05,0,RMSE,rmse,lower,0.23574409357673579,-0.23574409357673579,0.2418545849757855,mean,exp/integrated/sweeps/sberbank-housing/rla_rank2_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_rank2_noise1e-05-evaluation/0,False,814 +sweeps,sberbank-housing,rla_rank4_noise0.001,0,RMSE,rmse,lower,0.23544592963840097,-0.23544592963840097,0.23541433043431914,mean,exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.001-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.001-evaluation/0,False,888 +sweeps,sberbank-housing,rla_rank4_noise0.0001,0,RMSE,rmse,lower,0.2354418316334552,-0.2354418316334552,0.23162903716515967,mean,exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.0001-evaluation/0,False,666 +sweeps,sberbank-housing,rla_rank4_noise1e-05,0,RMSE,rmse,lower,0.23604436658345962,-0.23604436658345962,0.2328903381745147,mean,exp/integrated/sweeps/sberbank-housing/rla_rank4_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_rank4_noise1e-05-evaluation/0,False,666 +sweeps,sberbank-housing,rla_rank8_noise0.001,0,RMSE,rmse,lower,0.23625085686725053,-0.23625085686725053,0.2355661245169954,mean,exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.001-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.001-evaluation/0,False,888 +sweeps,sberbank-housing,rla_rank8_noise0.0001,0,RMSE,rmse,lower,0.23560034396487695,-0.23560034396487695,0.2314192197875428,mean,exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.0001-evaluation/0,False,666 +sweeps,sberbank-housing,rla_rank8_noise1e-05,0,RMSE,rmse,lower,0.23553038961691936,-0.23553038961691936,0.2411268296110548,mean,exp/integrated/sweeps/sberbank-housing/rla_rank8_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_rank8_noise1e-05-evaluation/0,False,814 +sweeps,sberbank-housing,esam_rho0.001,0,RMSE,rmse,lower,0.23467774706155475,-0.23467774706155475,0.23716614057842658,mean,exp/integrated/sweeps/sberbank-housing/esam_rho0.001-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/esam_rho0.001-evaluation/0,False,888 +sweeps,sberbank-housing,esam_rho0.0025,0,RMSE,rmse,lower,0.23474623366768377,-0.23474623366768377,0.23936563514587833,mean,exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-evaluation/0,False,814 +sweeps,sberbank-housing,esam_rho0.005,0,RMSE,rmse,lower,0.23533073111501152,-0.23533073111501152,0.24162357572341994,mean,exp/integrated/sweeps/sberbank-housing/esam_rho0.005-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/esam_rho0.005-evaluation/0,False,888 +sweeps,sberbank-housing,esam_rho0.01,0,RMSE,rmse,lower,0.2345959019738195,-0.2345959019738195,0.23852799100834732,mean,exp/integrated/sweeps/sberbank-housing/esam_rho0.01-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/esam_rho0.01-evaluation/0,False,814 +sweeps,sberbank-housing,mfb_keep0.7,0,RMSE,rmse,lower,0.23535581244592554,-0.23535581244592554,0.23562693018337708,mean,exp/integrated/sweeps/sberbank-housing/mfb_keep0.7-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/mfb_keep0.7-evaluation/0,False,888 +sweeps,sberbank-housing,mfb_keep0.8,0,RMSE,rmse,lower,0.2341279953511937,-0.2341279953511937,0.23402994505001048,mean,exp/integrated/sweeps/sberbank-housing/mfb_keep0.8-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/mfb_keep0.8-evaluation/0,False,888 +sweeps,sberbank-housing,mfb_keep0.9,0,RMSE,rmse,lower,0.23520949733542967,-0.23520949733542967,0.23880983570262945,mean,exp/integrated/sweeps/sberbank-housing/mfb_keep0.9-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/mfb_keep0.9-evaluation/0,False,1184 +sweeps,sberbank-housing,mfb_keep0.95,0,RMSE,rmse,lower,0.2352128391676174,-0.2352128391676174,0.23206813724553327,mean,exp/integrated/sweeps/sberbank-housing/mfb_keep0.95-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/mfb_keep0.95-evaluation/0,False,888 +sweeps,sberbank-housing,cf_fisd_lambda0.05,0,RMSE,rmse,lower,0.2359593327633197,-0.2359593327633197,0.23536697907032925,mean,exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.05-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.05-evaluation/0,False,666 +sweeps,sberbank-housing,cf_fisd_lambda0.1,0,RMSE,rmse,lower,0.23523413233394516,-0.23523413233394516,0.235149605552611,mean,exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.1-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.1-evaluation/0,False,666 +sweeps,sberbank-housing,cf_fisd_lambda0.2,0,RMSE,rmse,lower,0.23533942163580152,-0.23533942163580152,0.24087782495581203,mean,exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.2-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.2-evaluation/0,False,814 +sweeps,sberbank-housing,rla_esam,0,RMSE,rmse,lower,0.23528677495791916,-0.23528677495791916,0.23252163677391782,mean,exp/integrated/sweeps/sberbank-housing/rla_esam-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_esam-evaluation/0,False,666 +sweeps,sberbank-housing,rla_mfb,0,RMSE,rmse,lower,0.23551030167646542,-0.23551030167646542,0.23422079576562096,mean,exp/integrated/sweeps/sberbank-housing/rla_mfb-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_mfb-evaluation/0,False,888 +sweeps,sberbank-housing,rla_cf_fisd,0,RMSE,rmse,lower,0.23507577716825093,-0.23507577716825093,0.2401363436403948,mean,exp/integrated/sweeps/sberbank-housing/rla_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_cf_fisd-evaluation/0,False,888 +sweeps,sberbank-housing,esam_mfb,0,RMSE,rmse,lower,0.23493017564475285,-0.23493017564475285,0.23218164932047,mean,exp/integrated/sweeps/sberbank-housing/esam_mfb-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/esam_mfb-evaluation/0,False,888 +sweeps,sberbank-housing,esam_cf_fisd,0,RMSE,rmse,lower,0.2345315410452097,-0.2345315410452097,0.249503671176673,mean,exp/integrated/sweeps/sberbank-housing/esam_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/esam_cf_fisd-evaluation/0,False,1406 +sweeps,sberbank-housing,mfb_cf_fisd,0,RMSE,rmse,lower,0.23508488120210985,-0.23508488120210985,0.23568789402876444,mean,exp/integrated/sweeps/sberbank-housing/mfb_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/mfb_cf_fisd-evaluation/0,False,888 +sweeps,sberbank-housing,all_four_combined,0,RMSE,rmse,lower,0.2348792849855834,-0.2348792849855834,0.2326272249658597,mean,exp/integrated/sweeps/sberbank-housing/all_four_combined-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/all_four_combined-evaluation/0,False,666 +sweeps,sberbank-housing,all_minus_rla,0,RMSE,rmse,lower,0.23564260539703746,-0.23564260539703746,0.2337545527178023,mean,exp/integrated/sweeps/sberbank-housing/all_minus_rla-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/all_minus_rla-evaluation/0,False,666 +sweeps,sberbank-housing,all_minus_esam,0,RMSE,rmse,lower,0.23494022877441162,-0.23494022877441162,0.2319230328213383,mean,exp/integrated/sweeps/sberbank-housing/all_minus_esam-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/all_minus_esam-evaluation/0,False,666 +sweeps,sberbank-housing,all_minus_mfb,0,RMSE,rmse,lower,0.23539867958344296,-0.23539867958344296,0.2332565482846656,mean,exp/integrated/sweeps/sberbank-housing/all_minus_mfb-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/all_minus_mfb-evaluation/0,False,666 +sweeps,sberbank-housing,all_minus_cf_fisd,0,RMSE,rmse,lower,0.23469635866718716,-0.23469635866718716,0.23237236681744675,mean,exp/integrated/sweeps/sberbank-housing/all_minus_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/all_minus_cf_fisd-evaluation/0,False,666 +sweeps,sberbank-housing,mfb_keep0.7,0,RMSE,rmse,lower,0.24062365494389312,-0.24062365494389312,0.24557535127152327,best-head,exp/integrated/sweeps/sberbank-housing/mfb_keep0.7-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/mfb_keep0.7-best-head-evaluation/0,False,888 +sweeps,sberbank-housing,mfb_keep0.8,0,RMSE,rmse,lower,0.2380346366558266,-0.2380346366558266,0.2579307429494123,best-head,exp/integrated/sweeps/sberbank-housing/mfb_keep0.8-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/mfb_keep0.8-best-head-evaluation/0,False,888 +sweeps,sberbank-housing,mfb_keep0.7,0,RMSE,rmse,lower,0.23323099362803706,-0.23323099362803706,0.2357129689640159,greedy-heads,exp/integrated/sweeps/sberbank-housing/mfb_keep0.7-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/mfb_keep0.7-greedy-heads-evaluation/0,False,888 +sweeps,sberbank-housing,mfb_keep0.8,0,RMSE,rmse,lower,0.23237596587359466,-0.23237596587359466,0.23064696162155762,greedy-heads,exp/integrated/sweeps/sberbank-housing/mfb_keep0.8-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/mfb_keep0.8-greedy-heads-evaluation/0,False,888 +sweeps,sberbank-housing,mfb_keep0.9,0,RMSE,rmse,lower,0.24356476580119216,-0.24356476580119216,0.2459548830871388,best-head,exp/integrated/sweeps/sberbank-housing/mfb_keep0.9-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/mfb_keep0.9-best-head-evaluation/0,False,1184 +sweeps,sberbank-housing,mfb_keep0.9,0,RMSE,rmse,lower,0.233840053737128,-0.233840053737128,0.23548819505307594,greedy-heads,exp/integrated/sweeps/sberbank-housing/mfb_keep0.9-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/mfb_keep0.9-greedy-heads-evaluation/0,False,1184 +sweeps,sberbank-housing,rla_rank2_noise1e-05,0,RMSE,rmse,lower,0.2389693720816827,-0.2389693720816827,0.24127702155259925,best-head,exp/integrated/sweeps/sberbank-housing/rla_rank2_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_rank2_noise1e-05-best-head-evaluation/0,False,814 +sweeps,sberbank-housing,rla_rank2_noise1e-05,0,RMSE,rmse,lower,0.23334154528971895,-0.23334154528971895,0.23451478286562477,greedy-heads,exp/integrated/sweeps/sberbank-housing/rla_rank2_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_rank2_noise1e-05-greedy-heads-evaluation/0,False,814 +sweeps,sberbank-housing,rla_rank2_noise0.001,0,RMSE,rmse,lower,0.2370483756746748,-0.2370483756746748,0.23920040609688478,best-head,exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.001-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.001-best-head-evaluation/0,False,814 +sweeps,sberbank-housing,rla_rank2_noise0.001,0,RMSE,rmse,lower,0.2328613038082034,-0.2328613038082034,0.23498425768312764,greedy-heads,exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.001-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.001-greedy-heads-evaluation/0,False,814 +sweeps,sberbank-housing,rla_rank2_noise0.0001,0,RMSE,rmse,lower,0.23946739670755524,-0.23946739670755524,0.25239608328273316,best-head,exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.0001-best-head-evaluation/0,False,962 +sweeps,sberbank-housing,rla_rank2_noise0.0001,0,RMSE,rmse,lower,0.23332344829675253,-0.23332344829675253,0.24781019077451777,greedy-heads,exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.0001-greedy-heads-evaluation/0,False,962 +sweeps,sberbank-housing,esam_rho0.01,0,RMSE,rmse,lower,0.23855772550880586,-0.23855772550880586,0.24435949051951408,best-head,exp/integrated/sweeps/sberbank-housing/esam_rho0.01-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/esam_rho0.01-best-head-evaluation/0,False,814 +sweeps,sberbank-housing,esam_rho0.01,0,RMSE,rmse,lower,0.23321113093450146,-0.23321113093450146,0.23887208479744804,greedy-heads,exp/integrated/sweeps/sberbank-housing/esam_rho0.01-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/esam_rho0.01-greedy-heads-evaluation/0,False,814 +sweeps,sberbank-housing,esam_rho0.0025,0,RMSE,rmse,lower,0.2377136850270048,-0.2377136850270048,0.262345673404661,best-head,exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-best-head-evaluation/0,False,814 +sweeps,sberbank-housing,esam_rho0.0025,0,RMSE,rmse,lower,0.23295802298295287,-0.23295802298295287,0.23903366020873654,greedy-heads,exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-greedy-heads-evaluation/0,False,814 +sweeps,sberbank-housing,esam_rho0.005,0,RMSE,rmse,lower,0.2398028724080995,-0.2398028724080995,0.23711371881317633,best-head,exp/integrated/sweeps/sberbank-housing/esam_rho0.005-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/esam_rho0.005-best-head-evaluation/0,False,888 +sweeps,sberbank-housing,esam_rho0.001,0,RMSE,rmse,lower,0.23949031043262045,-0.23949031043262045,0.24155888192007524,best-head,exp/integrated/sweeps/sberbank-housing/esam_rho0.001-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/esam_rho0.001-best-head-evaluation/0,False,888 +sweeps,sberbank-housing,esam_rho0.005,0,RMSE,rmse,lower,0.23380275644783471,-0.23380275644783471,0.24257603366460162,greedy-heads,exp/integrated/sweeps/sberbank-housing/esam_rho0.005-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/esam_rho0.005-greedy-heads-evaluation/0,False,888 +sweeps,sberbank-housing,esam_rho0.001,0,RMSE,rmse,lower,0.23316146280351166,-0.23316146280351166,0.23452524296935331,greedy-heads,exp/integrated/sweeps/sberbank-housing/esam_rho0.001-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/esam_rho0.001-greedy-heads-evaluation/0,False,888 +sweeps,sberbank-housing,rla_rank4_noise1e-05,0,RMSE,rmse,lower,0.23804322063719607,-0.23804322063719607,0.24414388272972984,best-head,exp/integrated/sweeps/sberbank-housing/rla_rank4_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_rank4_noise1e-05-best-head-evaluation/0,False,666 +sweeps,sberbank-housing,rla_rank4_noise1e-05,0,RMSE,rmse,lower,0.2335869733918295,-0.2335869733918295,0.2353936312104005,greedy-heads,exp/integrated/sweeps/sberbank-housing/rla_rank4_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_rank4_noise1e-05-greedy-heads-evaluation/0,False,666 +sweeps,sberbank-housing,rla_rank4_noise0.0001,0,RMSE,rmse,lower,0.23765212789668771,-0.23765212789668771,0.24778267915496135,best-head,exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.0001-best-head-evaluation/0,False,666 +sweeps,sberbank-housing,rla_rank4_noise0.0001,0,RMSE,rmse,lower,0.2336025622310011,-0.2336025622310011,0.23389241285301454,greedy-heads,exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.0001-greedy-heads-evaluation/0,False,666 +sweeps,sberbank-housing,rla_rank4_noise0.001,0,RMSE,rmse,lower,0.23958569850307038,-0.23958569850307038,0.2417253037065434,best-head,exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.001-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.001-best-head-evaluation/0,False,888 +sweeps,sberbank-housing,rla_rank4_noise0.001,0,RMSE,rmse,lower,0.23389653800469046,-0.23389653800469046,0.23994098363644276,greedy-heads,exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.001-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.001-greedy-heads-evaluation/0,False,888 +sweeps,sberbank-housing,mfb_keep0.95,0,RMSE,rmse,lower,0.23835928999137548,-0.23835928999137548,0.27696045729988356,best-head,exp/integrated/sweeps/sberbank-housing/mfb_keep0.95-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/mfb_keep0.95-best-head-evaluation/0,False,888 +sweeps,sberbank-housing,mfb_keep0.95,0,RMSE,rmse,lower,0.23254505067097037,-0.23254505067097037,0.23506758402869707,greedy-heads,exp/integrated/sweeps/sberbank-housing/mfb_keep0.95-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/mfb_keep0.95-greedy-heads-evaluation/0,False,888 +sweeps,sberbank-housing,cf_fisd_lambda0.05,0,RMSE,rmse,lower,0.2394238965987828,-0.2394238965987828,0.23525657161407812,best-head,exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.05-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.05-best-head-evaluation/0,False,666 +sweeps,sberbank-housing,cf_fisd_lambda0.05,0,RMSE,rmse,lower,0.23452905519184072,-0.23452905519184072,0.23270925018080982,greedy-heads,exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.05-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.05-greedy-heads-evaluation/0,False,666 +sweeps,sberbank-housing,cf_fisd_lambda0.1,0,RMSE,rmse,lower,0.2388197411165918,-0.2388197411165918,0.2462523116085554,best-head,exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.1-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.1-best-head-evaluation/0,False,666 +sweeps,sberbank-housing,cf_fisd_lambda0.1,0,RMSE,rmse,lower,0.23327114512588593,-0.23327114512588593,0.23367141178994547,greedy-heads,exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.1-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.1-greedy-heads-evaluation/0,False,666 +sweeps,sberbank-housing,cf_fisd_lambda0.2,0,RMSE,rmse,lower,0.23866861156973368,-0.23866861156973368,0.2497089599248454,best-head,exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.2-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.2-best-head-evaluation/0,False,814 +sweeps,sberbank-housing,cf_fisd_lambda0.2,0,RMSE,rmse,lower,0.23337569983908155,-0.23337569983908155,0.244512625524316,greedy-heads,exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.2-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.2-greedy-heads-evaluation/0,False,814 +sweeps,sberbank-housing,rla_rank8_noise0.0001,0,RMSE,rmse,lower,0.23612384794902308,-0.23612384794902308,0.25131993238610145,best-head,exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.0001-best-head-evaluation/0,False,666 +sweeps,sberbank-housing,rla_rank8_noise0.0001,0,RMSE,rmse,lower,0.23331262295778735,-0.23331262295778735,0.2320760028721284,greedy-heads,exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.0001-greedy-heads-evaluation/0,False,666 +sweeps,sberbank-housing,rla_rank8_noise1e-05,0,RMSE,rmse,lower,0.23899585630204528,-0.23899585630204528,0.25744912519923097,best-head,exp/integrated/sweeps/sberbank-housing/rla_rank8_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_rank8_noise1e-05-best-head-evaluation/0,False,814 +sweeps,sberbank-housing,rla_rank8_noise1e-05,0,RMSE,rmse,lower,0.23346652550110514,-0.23346652550110514,0.24465339979644968,greedy-heads,exp/integrated/sweeps/sberbank-housing/rla_rank8_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_rank8_noise1e-05-greedy-heads-evaluation/0,False,814 +sweeps,sberbank-housing,rla_rank8_noise0.001,0,RMSE,rmse,lower,0.239120553017687,-0.239120553017687,0.23826554512424344,best-head,exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.001-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.001-best-head-evaluation/0,False,888 +sweeps,sberbank-housing,rla_rank8_noise0.001,0,RMSE,rmse,lower,0.23353637226653048,-0.23353637226653048,0.23842460978700958,greedy-heads,exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.001-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.001-greedy-heads-evaluation/0,False,888 +sweeps,sberbank-housing,mfb_cf_fisd,0,RMSE,rmse,lower,0.23765498863853576,-0.23765498863853576,0.2536206620016002,best-head,exp/integrated/sweeps/sberbank-housing/mfb_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/mfb_cf_fisd-best-head-evaluation/0,False,888 +sweeps,sberbank-housing,mfb_cf_fisd,0,RMSE,rmse,lower,0.23320900639668657,-0.23320900639668657,0.23362227205150313,greedy-heads,exp/integrated/sweeps/sberbank-housing/mfb_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/mfb_cf_fisd-greedy-heads-evaluation/0,False,888 +sweeps,sberbank-housing,rla_mfb,0,RMSE,rmse,lower,0.2399809438744672,-0.2399809438744672,0.24194702392020134,best-head,exp/integrated/sweeps/sberbank-housing/rla_mfb-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_mfb-best-head-evaluation/0,False,888 +sweeps,sberbank-housing,rla_mfb,0,RMSE,rmse,lower,0.23358161473727912,-0.23358161473727912,0.2330478525093287,greedy-heads,exp/integrated/sweeps/sberbank-housing/rla_mfb-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_mfb-greedy-heads-evaluation/0,False,888 +sweeps,sberbank-housing,esam_mfb,0,RMSE,rmse,lower,0.23962393015909736,-0.23962393015909736,0.24513925399849143,best-head,exp/integrated/sweeps/sberbank-housing/esam_mfb-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/esam_mfb-best-head-evaluation/0,False,888 +sweeps,sberbank-housing,esam_mfb,0,RMSE,rmse,lower,0.2336977870530739,-0.2336977870530739,0.23066386352840565,greedy-heads,exp/integrated/sweeps/sberbank-housing/esam_mfb-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/esam_mfb-greedy-heads-evaluation/0,False,888 +sweeps,sberbank-housing,rla_cf_fisd,0,RMSE,rmse,lower,0.23788056532412807,-0.23788056532412807,0.24833117735063384,best-head,exp/integrated/sweeps/sberbank-housing/rla_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_cf_fisd-best-head-evaluation/0,False,888 +sweeps,sberbank-housing,rla_cf_fisd,0,RMSE,rmse,lower,0.233603375533187,-0.233603375533187,0.23596160620261294,greedy-heads,exp/integrated/sweeps/sberbank-housing/rla_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_cf_fisd-greedy-heads-evaluation/0,False,888 +sweeps,sberbank-housing,all_minus_rla,0,RMSE,rmse,lower,0.2383111872393019,-0.2383111872393019,0.2360172512727007,best-head,exp/integrated/sweeps/sberbank-housing/all_minus_rla-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/all_minus_rla-best-head-evaluation/0,False,666 +sweeps,sberbank-housing,all_minus_rla,0,RMSE,rmse,lower,0.23419188657775022,-0.23419188657775022,0.23442550750747457,greedy-heads,exp/integrated/sweeps/sberbank-housing/all_minus_rla-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/all_minus_rla-greedy-heads-evaluation/0,False,666 +sweeps,sberbank-housing,all_minus_esam,0,RMSE,rmse,lower,0.23762339316571812,-0.23762339316571812,0.2548487995005296,best-head,exp/integrated/sweeps/sberbank-housing/all_minus_esam-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/all_minus_esam-best-head-evaluation/0,False,666 +sweeps,sberbank-housing,all_minus_esam,0,RMSE,rmse,lower,0.23366970594362974,-0.23366970594362974,0.23132655964186158,greedy-heads,exp/integrated/sweeps/sberbank-housing/all_minus_esam-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/all_minus_esam-greedy-heads-evaluation/0,False,666 +sweeps,sberbank-housing,rla_esam,0,RMSE,rmse,lower,0.2398484083586863,-0.2398484083586863,0.2374880282622196,best-head,exp/integrated/sweeps/sberbank-housing/rla_esam-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_esam-best-head-evaluation/0,False,666 +sweeps,sberbank-housing,rla_esam,0,RMSE,rmse,lower,0.2336232925794531,-0.2336232925794531,0.233028022164673,greedy-heads,exp/integrated/sweeps/sberbank-housing/rla_esam-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_esam-greedy-heads-evaluation/0,False,666 +sweeps,sberbank-housing,esam_cf_fisd,0,RMSE,rmse,lower,0.2404271956933659,-0.2404271956933659,0.25751796428274565,best-head,exp/integrated/sweeps/sberbank-housing/esam_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/esam_cf_fisd-best-head-evaluation/0,False,1406 +sweeps,sberbank-housing,esam_cf_fisd,0,RMSE,rmse,lower,0.23329227219621285,-0.23329227219621285,0.23856120001818734,greedy-heads,exp/integrated/sweeps/sberbank-housing/esam_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/esam_cf_fisd-greedy-heads-evaluation/0,False,1406 +sweeps,sberbank-housing,all_four_combined,0,RMSE,rmse,lower,0.2376718937442043,-0.2376718937442043,0.2539783962159738,best-head,exp/integrated/sweeps/sberbank-housing/all_four_combined-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/all_four_combined-best-head-evaluation/0,False,666 +sweeps,sberbank-housing,all_four_combined,0,RMSE,rmse,lower,0.23335850745168515,-0.23335850745168515,0.23263890687901315,greedy-heads,exp/integrated/sweeps/sberbank-housing/all_four_combined-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/all_four_combined-greedy-heads-evaluation/0,False,666 +sweeps,sberbank-housing,all_minus_cf_fisd,0,RMSE,rmse,lower,0.23808479027162685,-0.23808479027162685,0.27950919024280946,best-head,exp/integrated/sweeps/sberbank-housing/all_minus_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/all_minus_cf_fisd-best-head-evaluation/0,False,666 +sweeps,sberbank-housing,all_minus_cf_fisd,0,RMSE,rmse,lower,0.23321538793887137,-0.23321538793887137,0.23519795893846665,greedy-heads,exp/integrated/sweeps/sberbank-housing/all_minus_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/all_minus_cf_fisd-greedy-heads-evaluation/0,False,666 +sweeps,sberbank-housing,all_minus_mfb,0,RMSE,rmse,lower,0.23769862439897374,-0.23769862439897374,0.24291600363173604,best-head,exp/integrated/sweeps/sberbank-housing/all_minus_mfb-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/all_minus_mfb-best-head-evaluation/0,False,666 +sweeps,sberbank-housing,all_minus_mfb,0,RMSE,rmse,lower,0.23349088968550719,-0.23349088968550719,0.23959467004131907,greedy-heads,exp/integrated/sweeps/sberbank-housing/all_minus_mfb-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/all_minus_mfb-greedy-heads-evaluation/0,False,666 +sweeps,ecom-offers,rla_rank2_noise0.001,0,AUROC,roc-auc,higher,0.6226233688882503,0.6226233688882503,0.5911217830862241,mean,exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.001-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.001-evaluation/0,False,214 +sweeps,ecom-offers,rla_rank2_noise0.0001,0,AUROC,roc-auc,higher,0.6227001707049361,0.6227001707049361,0.5911765632808944,mean,exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.0001-evaluation/0,False,214 +sweeps,ecom-offers,rla_rank2_noise1e-05,0,AUROC,roc-auc,higher,0.6226529287429354,0.6226529287429354,0.5911485904814773,mean,exp/integrated/sweeps/ecom-offers/rla_rank2_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_rank2_noise1e-05-evaluation/0,False,214 +sweeps,ecom-offers,rla_rank4_noise0.001,0,AUROC,roc-auc,higher,0.622598916870318,0.622598916870318,0.5911027119909538,mean,exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.001-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.001-evaluation/0,False,214 +sweeps,ecom-offers,rla_rank4_noise0.0001,0,AUROC,roc-auc,higher,0.6226365549199251,0.6226365549199251,0.5911320299643263,mean,exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.0001-evaluation/0,False,214 +sweeps,ecom-offers,rla_rank4_noise1e-05,0,AUROC,roc-auc,higher,0.6226645595746754,0.6226645595746754,0.5911489283247057,mean,exp/integrated/sweeps/ecom-offers/rla_rank4_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_rank4_noise1e-05-evaluation/0,False,214 +sweeps,ecom-offers,rla_rank8_noise0.001,0,AUROC,roc-auc,higher,0.6225384372823319,0.6225384372823319,0.591035974005685,mean,exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.001-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.001-evaluation/0,False,214 +sweeps,ecom-offers,rla_rank8_noise0.0001,0,AUROC,roc-auc,higher,0.6225693459597093,0.6225693459597093,0.5910522493707531,mean,exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.0001-evaluation/0,False,214 +sweeps,ecom-offers,rla_rank8_noise1e-05,0,AUROC,roc-auc,higher,0.6226251931156621,0.6226251931156621,0.5910938288836818,mean,exp/integrated/sweeps/ecom-offers/rla_rank8_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_rank8_noise1e-05-evaluation/0,False,214 +sweeps,ecom-offers,esam_rho0.001,0,AUROC,roc-auc,higher,0.6227874903903754,0.6227874903903754,0.5911993816464689,mean,exp/integrated/sweeps/ecom-offers/esam_rho0.001-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/esam_rho0.001-evaluation/0,False,321 +sweeps,ecom-offers,esam_rho0.0025,0,AUROC,roc-auc,higher,0.6227333347907498,0.6227333347907498,0.5911601949314514,mean,exp/integrated/sweeps/ecom-offers/esam_rho0.0025-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/esam_rho0.0025-evaluation/0,False,321 +sweeps,ecom-offers,esam_rho0.005,0,AUROC,roc-auc,higher,0.6227207494642631,0.6227207494642631,0.5911612394559284,mean,exp/integrated/sweeps/ecom-offers/esam_rho0.005-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/esam_rho0.005-evaluation/0,False,321 +sweeps,ecom-offers,esam_rho0.01,0,AUROC,roc-auc,higher,0.6227573335158101,0.6227573335158101,0.5911827002496308,mean,exp/integrated/sweeps/ecom-offers/esam_rho0.01-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/esam_rho0.01-evaluation/0,False,321 +sweeps,ecom-offers,mfb_keep0.7,0,AUROC,roc-auc,higher,0.6291199291618959,0.6291199291618959,0.5961977102213372,mean,exp/integrated/sweeps/ecom-offers/mfb_keep0.7-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/mfb_keep0.7-evaluation/0,False,428 +sweeps,ecom-offers,mfb_keep0.8,0,AUROC,roc-auc,higher,0.6293880905914133,0.6293880905914133,0.5962685705138886,mean,exp/integrated/sweeps/ecom-offers/mfb_keep0.8-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/mfb_keep0.8-evaluation/0,False,321 +sweeps,ecom-offers,mfb_keep0.9,0,AUROC,roc-auc,higher,0.6250090898119413,0.6250090898119413,0.5927112610884333,mean,exp/integrated/sweeps/ecom-offers/mfb_keep0.9-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/mfb_keep0.9-evaluation/0,False,321 +sweeps,ecom-offers,mfb_keep0.95,0,AUROC,roc-auc,higher,0.6252559501617828,0.6252559501617828,0.5929062493223765,mean,exp/integrated/sweeps/ecom-offers/mfb_keep0.95-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/mfb_keep0.95-evaluation/0,False,321 +sweeps,ecom-offers,cf_fisd_lambda0.05,0,AUROC,roc-auc,higher,0.6228186386127663,0.6228186386127663,0.5911736900637132,mean,exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.05-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.05-evaluation/0,False,321 +sweeps,ecom-offers,cf_fisd_lambda0.1,0,AUROC,roc-auc,higher,0.6228570653182646,0.6228570653182646,0.5912083608375949,mean,exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.1-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.1-evaluation/0,False,321 +sweeps,ecom-offers,cf_fisd_lambda0.2,0,AUROC,roc-auc,higher,0.622913421046708,0.622913421046708,0.5913362546461658,mean,exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.2-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.2-evaluation/0,False,214 +sweeps,ecom-offers,rla_esam,0,AUROC,roc-auc,higher,0.6225317005394864,0.6225317005394864,0.5910259812848737,mean,exp/integrated/sweeps/ecom-offers/rla_esam-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_esam-evaluation/0,False,214 +sweeps,ecom-offers,rla_mfb,0,AUROC,roc-auc,higher,0.6244884295114916,0.6244884295114916,0.5923593741199417,mean,exp/integrated/sweeps/ecom-offers/rla_mfb-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_mfb-evaluation/0,False,321 +sweeps,ecom-offers,rla_cf_fisd,0,AUROC,roc-auc,higher,0.6227441474841354,0.6227441474841354,0.5911932818714825,mean,exp/integrated/sweeps/ecom-offers/rla_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_cf_fisd-evaluation/0,False,214 +sweeps,ecom-offers,esam_mfb,0,AUROC,roc-auc,higher,0.6249124647240485,0.6249124647240485,0.5926966501436739,mean,exp/integrated/sweeps/ecom-offers/esam_mfb-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/esam_mfb-evaluation/0,False,321 +sweeps,ecom-offers,esam_cf_fisd,0,AUROC,roc-auc,higher,0.6227702320934685,0.6227702320934685,0.5911624823470715,mean,exp/integrated/sweeps/ecom-offers/esam_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/esam_cf_fisd-evaluation/0,False,321 +sweeps,ecom-offers,mfb_cf_fisd,0,AUROC,roc-auc,higher,0.6250550271749447,0.6250550271749447,0.592670158895295,mean,exp/integrated/sweeps/ecom-offers/mfb_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/mfb_cf_fisd-evaluation/0,False,321 +sweeps,ecom-offers,all_four_combined,0,AUROC,roc-auc,higher,0.6244437709503304,0.6244437709503304,0.592244305956134,mean,exp/integrated/sweeps/ecom-offers/all_four_combined-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/all_four_combined-evaluation/0,False,321 +sweeps,ecom-offers,all_minus_rla,0,AUROC,roc-auc,higher,0.6249365260993432,0.6249365260993432,0.5925964470819166,mean,exp/integrated/sweeps/ecom-offers/all_minus_rla-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/all_minus_rla-evaluation/0,False,321 +sweeps,ecom-offers,all_minus_esam,0,AUROC,roc-auc,higher,0.6245232151327823,0.6245232151327823,0.5923207050179439,mean,exp/integrated/sweeps/ecom-offers/all_minus_esam-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/all_minus_esam-evaluation/0,False,321 +sweeps,ecom-offers,all_minus_mfb,0,AUROC,roc-auc,higher,0.6226455728683622,0.6226455728683622,0.5910976319446108,mean,exp/integrated/sweeps/ecom-offers/all_minus_mfb-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/all_minus_mfb-evaluation/0,False,214 +sweeps,ecom-offers,all_minus_cf_fisd,0,AUROC,roc-auc,higher,0.6244116424360375,0.6244116424360375,0.5923170352346188,mean,exp/integrated/sweeps/ecom-offers/all_minus_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/all_minus_cf_fisd-evaluation/0,False,321 +sweeps,ecom-offers,rla_rank2_noise0.001,0,AUROC,roc-auc,higher,0.6395828610304147,0.6395828610304147,0.601475038412775,best-head,exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.001-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.001-best-head-evaluation/0,False,214 +sweeps,ecom-offers,rla_rank2_noise0.0001,0,AUROC,roc-auc,higher,0.6396541901648656,0.6396541901648656,0.6015418321886687,best-head,exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.0001-best-head-evaluation/0,False,214 +sweeps,ecom-offers,rla_rank2_noise0.001,0,AUROC,roc-auc,higher,0.6395828610304147,0.6395828610304147,0.601475038412775,greedy-heads,exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.001-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.001-greedy-heads-evaluation/0,False,214 +sweeps,ecom-offers,rla_rank2_noise0.0001,0,AUROC,roc-auc,higher,0.6396541901648656,0.6396541901648656,0.6015418321886687,greedy-heads,exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.0001-greedy-heads-evaluation/0,False,214 +sweeps,ecom-offers,rla_rank2_noise1e-05,0,AUROC,roc-auc,higher,0.6395464280764918,0.6395464280764918,0.6014523595237629,best-head,exp/integrated/sweeps/ecom-offers/rla_rank2_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_rank2_noise1e-05-best-head-evaluation/0,False,214 +sweeps,ecom-offers,rla_rank2_noise1e-05,0,AUROC,roc-auc,higher,0.6395464280764918,0.6395464280764918,0.6014523595237629,greedy-heads,exp/integrated/sweeps/ecom-offers/rla_rank2_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_rank2_noise1e-05-greedy-heads-evaluation/0,False,214 +sweeps,ecom-offers,rla_rank4_noise0.001,0,AUROC,roc-auc,higher,0.6394467257565004,0.6394467257565004,0.6014832923257777,best-head,exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.001-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.001-best-head-evaluation/0,False,214 +sweeps,ecom-offers,rla_rank4_noise0.0001,0,AUROC,roc-auc,higher,0.6394530055211662,0.6394530055211662,0.6014026252811646,best-head,exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.0001-best-head-evaluation/0,False,214 +sweeps,ecom-offers,rla_rank4_noise0.001,0,AUROC,roc-auc,higher,0.6394467257565004,0.6394467257565004,0.6014832923257777,greedy-heads,exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.001-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.001-greedy-heads-evaluation/0,False,214 +sweeps,ecom-offers,rla_rank4_noise1e-05,0,AUROC,roc-auc,higher,0.6396100364908871,0.6396100364908871,0.6014864444960832,best-head,exp/integrated/sweeps/ecom-offers/rla_rank4_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_rank4_noise1e-05-best-head-evaluation/0,False,214 +sweeps,ecom-offers,rla_rank4_noise0.0001,0,AUROC,roc-auc,higher,0.6394530055211662,0.6394530055211662,0.6014026252811646,greedy-heads,exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.0001-greedy-heads-evaluation/0,False,214 +sweeps,ecom-offers,rla_rank4_noise1e-05,0,AUROC,roc-auc,higher,0.6396100364908871,0.6396100364908871,0.6014864444960832,greedy-heads,exp/integrated/sweeps/ecom-offers/rla_rank4_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_rank4_noise1e-05-greedy-heads-evaluation/0,False,214 +sweeps,ecom-offers,mfb_keep0.7,0,AUROC,roc-auc,higher,0.6547032702847417,0.6547032702847417,0.6118564029158264,best-head,exp/integrated/sweeps/ecom-offers/mfb_keep0.7-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/mfb_keep0.7-best-head-evaluation/0,False,428 +sweeps,ecom-offers,mfb_keep0.7,0,AUROC,roc-auc,higher,0.6547032702847417,0.6547032702847417,0.6118564029158264,greedy-heads,exp/integrated/sweeps/ecom-offers/mfb_keep0.7-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/mfb_keep0.7-greedy-heads-evaluation/0,False,428 +sweeps,ecom-offers,mfb_keep0.8,0,AUROC,roc-auc,higher,0.652377048657135,0.652377048657135,0.6066780069996902,best-head,exp/integrated/sweeps/ecom-offers/mfb_keep0.8-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/mfb_keep0.8-best-head-evaluation/0,False,321 +sweeps,ecom-offers,esam_rho0.001,0,AUROC,roc-auc,higher,0.6410241296714208,0.6410241296714208,0.6024966050474919,best-head,exp/integrated/sweeps/ecom-offers/esam_rho0.001-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/esam_rho0.001-best-head-evaluation/0,False,321 +sweeps,ecom-offers,mfb_keep0.9,0,AUROC,roc-auc,higher,0.6501781765294031,0.6501781765294031,0.6048261425892079,best-head,exp/integrated/sweeps/ecom-offers/mfb_keep0.9-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/mfb_keep0.9-best-head-evaluation/0,False,321 +sweeps,ecom-offers,esam_rho0.0025,0,AUROC,roc-auc,higher,0.64045684285539,0.64045684285539,0.6022469172053364,best-head,exp/integrated/sweeps/ecom-offers/esam_rho0.0025-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/esam_rho0.0025-best-head-evaluation/0,False,321 +sweeps,ecom-offers,esam_rho0.005,0,AUROC,roc-auc,higher,0.6406711951042247,0.6406711951042247,0.6023965414622969,best-head,exp/integrated/sweeps/ecom-offers/esam_rho0.005-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/esam_rho0.005-best-head-evaluation/0,False,321 +sweeps,ecom-offers,mfb_keep0.8,0,AUROC,roc-auc,higher,0.6540790454616192,0.6540790454616192,0.6102977368359448,greedy-heads,exp/integrated/sweeps/ecom-offers/mfb_keep0.8-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/mfb_keep0.8-greedy-heads-evaluation/0,False,321 +sweeps,ecom-offers,esam_rho0.01,0,AUROC,roc-auc,higher,0.6407133660825518,0.6407133660825518,0.6023744607727624,best-head,exp/integrated/sweeps/ecom-offers/esam_rho0.01-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/esam_rho0.01-best-head-evaluation/0,False,321 +sweeps,ecom-offers,esam_rho0.001,0,AUROC,roc-auc,higher,0.641845068859757,0.641845068859757,0.6041622806453656,greedy-heads,exp/integrated/sweeps/ecom-offers/esam_rho0.001-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/esam_rho0.001-greedy-heads-evaluation/0,False,321 +sweeps,ecom-offers,mfb_keep0.9,0,AUROC,roc-auc,higher,0.6506721330887921,0.6506721330887921,0.6064667434006787,greedy-heads,exp/integrated/sweeps/ecom-offers/mfb_keep0.9-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/mfb_keep0.9-greedy-heads-evaluation/0,False,321 +sweeps,ecom-offers,esam_rho0.0025,0,AUROC,roc-auc,higher,0.641480466607742,0.641480466607742,0.6039784195416061,greedy-heads,exp/integrated/sweeps/ecom-offers/esam_rho0.0025-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/esam_rho0.0025-greedy-heads-evaluation/0,False,321 +sweeps,ecom-offers,esam_rho0.005,0,AUROC,roc-auc,higher,0.6415752785241469,0.6415752785241469,0.6040898334194843,greedy-heads,exp/integrated/sweeps/ecom-offers/esam_rho0.005-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/esam_rho0.005-greedy-heads-evaluation/0,False,321 +sweeps,ecom-offers,esam_rho0.01,0,AUROC,roc-auc,higher,0.6415860912175324,0.6415860912175324,0.6040412707800087,greedy-heads,exp/integrated/sweeps/ecom-offers/esam_rho0.01-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/esam_rho0.01-greedy-heads-evaluation/0,False,321 +sweeps,ecom-offers,mfb_keep0.95,0,AUROC,roc-auc,higher,0.6503189442353507,0.6503189442353507,0.6050413611236272,best-head,exp/integrated/sweeps/ecom-offers/mfb_keep0.95-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/mfb_keep0.95-best-head-evaluation/0,False,321 +sweeps,ecom-offers,rla_rank8_noise0.001,0,AUROC,roc-auc,higher,0.6392187010153481,0.6392187010153481,0.6012738729168664,best-head,exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.001-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.001-best-head-evaluation/0,False,214 +sweeps,ecom-offers,rla_rank8_noise0.0001,0,AUROC,roc-auc,higher,0.639398116545268,0.639398116545268,0.6013710911801927,best-head,exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.0001-best-head-evaluation/0,False,214 +sweeps,ecom-offers,cf_fisd_lambda0.2,0,AUROC,roc-auc,higher,0.638077626500147,0.638077626500147,0.6005513130366958,best-head,exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.2-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.2-best-head-evaluation/0,False,214 +sweeps,ecom-offers,rla_rank8_noise0.001,0,AUROC,roc-auc,higher,0.6392187010153481,0.6392187010153481,0.6012738729168664,greedy-heads,exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.001-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.001-greedy-heads-evaluation/0,False,214 +sweeps,ecom-offers,cf_fisd_lambda0.05,0,AUROC,roc-auc,higher,0.6410202011331968,0.6410202011331968,0.6024701571918213,best-head,exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.05-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.05-best-head-evaluation/0,False,321 +sweeps,ecom-offers,rla_rank8_noise0.0001,0,AUROC,roc-auc,higher,0.639398116545268,0.639398116545268,0.6013710911801927,greedy-heads,exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.0001-greedy-heads-evaluation/0,False,214 +sweeps,ecom-offers,cf_fisd_lambda0.1,0,AUROC,roc-auc,higher,0.640920668337369,0.640920668337369,0.6022998594088651,best-head,exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.1-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.1-best-head-evaluation/0,False,321 +sweeps,ecom-offers,mfb_keep0.95,0,AUROC,roc-auc,higher,0.6505655760961024,0.6505655760961024,0.6064442039882287,greedy-heads,exp/integrated/sweeps/ecom-offers/mfb_keep0.95-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/mfb_keep0.95-greedy-heads-evaluation/0,False,321 +sweeps,ecom-offers,cf_fisd_lambda0.05,0,AUROC,roc-auc,higher,0.6419149054445072,0.6419149054445072,0.6041224275423289,greedy-heads,exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.05-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.05-greedy-heads-evaluation/0,False,321 +sweeps,ecom-offers,cf_fisd_lambda0.1,0,AUROC,roc-auc,higher,0.6418041361448852,0.6418041361448852,0.6039617691395597,greedy-heads,exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.1-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.1-greedy-heads-evaluation/0,False,321 +sweeps,ecom-offers,rla_rank8_noise1e-05,0,AUROC,roc-auc,higher,0.6393903110631306,0.6393903110631306,0.6013715932958167,best-head,exp/integrated/sweeps/ecom-offers/rla_rank8_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_rank8_noise1e-05-best-head-evaluation/0,False,214 +sweeps,ecom-offers,cf_fisd_lambda0.2,0,AUROC,roc-auc,higher,0.6387008784019832,0.6387008784019832,0.6023730040175572,greedy-heads,exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.2-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.2-greedy-heads-evaluation/0,False,214 +sweeps,ecom-offers,rla_rank8_noise1e-05,0,AUROC,roc-auc,higher,0.6393903110631306,0.6393903110631306,0.6013715932958167,greedy-heads,exp/integrated/sweeps/ecom-offers/rla_rank8_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_rank8_noise1e-05-greedy-heads-evaluation/0,False,214 +sweeps,ecom-offers,mfb_cf_fisd,0,AUROC,roc-auc,higher,0.6477890245840551,0.6477890245840551,0.6045088303107439,best-head,exp/integrated/sweeps/ecom-offers/mfb_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/mfb_cf_fisd-best-head-evaluation/0,False,321 +sweeps,ecom-offers,mfb_cf_fisd,0,AUROC,roc-auc,higher,0.6506783465179156,0.6506783465179156,0.6065618416202598,greedy-heads,exp/integrated/sweeps/ecom-offers/mfb_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/mfb_cf_fisd-greedy-heads-evaluation/0,False,321 +sweeps,ecom-offers,esam_mfb,0,AUROC,roc-auc,higher,0.6500013075472437,0.6500013075472437,0.6046018053871005,best-head,exp/integrated/sweeps/ecom-offers/esam_mfb-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/esam_mfb-best-head-evaluation/0,False,321 +sweeps,ecom-offers,rla_mfb,0,AUROC,roc-auc,higher,0.6500073735640511,0.6500073735640511,0.6048362127969982,best-head,exp/integrated/sweeps/ecom-offers/rla_mfb-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_mfb-best-head-evaluation/0,False,321 +sweeps,ecom-offers,rla_cf_fisd,0,AUROC,roc-auc,higher,0.6377017988002615,0.6377017988002615,0.6003639247256524,best-head,exp/integrated/sweeps/ecom-offers/rla_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_cf_fisd-best-head-evaluation/0,False,214 +sweeps,ecom-offers,esam_mfb,0,AUROC,roc-auc,higher,0.650368091501537,0.650368091501537,0.6061843343570334,greedy-heads,exp/integrated/sweeps/ecom-offers/esam_mfb-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/esam_mfb-greedy-heads-evaluation/0,False,321 +sweeps,ecom-offers,rla_mfb,0,AUROC,roc-auc,higher,0.6505360715210358,0.6505360715210358,0.6064993654188358,greedy-heads,exp/integrated/sweeps/ecom-offers/rla_mfb-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_mfb-greedy-heads-evaluation/0,False,321 +sweeps,ecom-offers,esam_cf_fisd,0,AUROC,roc-auc,higher,0.6410485079831951,0.6410485079831951,0.6024310820580536,best-head,exp/integrated/sweeps/ecom-offers/esam_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/esam_cf_fisd-best-head-evaluation/0,False,321 +sweeps,ecom-offers,rla_cf_fisd,0,AUROC,roc-auc,higher,0.6386263098818863,0.6386263098818863,0.6021922454924368,greedy-heads,exp/integrated/sweeps/ecom-offers/rla_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_cf_fisd-greedy-heads-evaluation/0,False,214 +sweeps,ecom-offers,esam_cf_fisd,0,AUROC,roc-auc,higher,0.6417659932080955,0.6417659932080955,0.6040036864957168,greedy-heads,exp/integrated/sweeps/ecom-offers/esam_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/esam_cf_fisd-greedy-heads-evaluation/0,False,321 +sweeps,ecom-offers,all_minus_rla,0,AUROC,roc-auc,higher,0.6475121989956593,0.6475121989956593,0.6043243524106316,best-head,exp/integrated/sweeps/ecom-offers/all_minus_rla-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/all_minus_rla-best-head-evaluation/0,False,321 +sweeps,ecom-offers,rla_esam,0,AUROC,roc-auc,higher,0.6394659925462143,0.6394659925462143,0.601268355843962,best-head,exp/integrated/sweeps/ecom-offers/rla_esam-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_esam-best-head-evaluation/0,False,214 +sweeps,ecom-offers,rla_esam,0,AUROC,roc-auc,higher,0.6394659925462143,0.6394659925462143,0.601268355843962,greedy-heads,exp/integrated/sweeps/ecom-offers/rla_esam-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_esam-greedy-heads-evaluation/0,False,214 +sweeps,ecom-offers,all_minus_rla,0,AUROC,roc-auc,higher,0.6504789566191646,0.6504789566191646,0.6063598206681031,greedy-heads,exp/integrated/sweeps/ecom-offers/all_minus_rla-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/all_minus_rla-greedy-heads-evaluation/0,False,321 +sweeps,ecom-offers,all_minus_esam,0,AUROC,roc-auc,higher,0.6476721782117019,0.6476721782117019,0.6040083543113315,best-head,exp/integrated/sweeps/ecom-offers/all_minus_esam-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/all_minus_esam-best-head-evaluation/0,False,321 +sweeps,ecom-offers,all_minus_esam,0,AUROC,roc-auc,higher,0.6504598446123828,0.6504598446123828,0.6064818006704371,greedy-heads,exp/integrated/sweeps/ecom-offers/all_minus_esam-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/all_minus_esam-greedy-heads-evaluation/0,False,321 +sweeps,ecom-offers,all_four_combined,0,AUROC,roc-auc,higher,0.6479868261151002,0.6479868261151002,0.6047175337399694,best-head,exp/integrated/sweeps/ecom-offers/all_four_combined-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/all_four_combined-best-head-evaluation/0,False,321 +sweeps,ecom-offers,all_minus_mfb,0,AUROC,roc-auc,higher,0.637709740638791,0.637709740638791,0.6004564813723224,best-head,exp/integrated/sweeps/ecom-offers/all_minus_mfb-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/all_minus_mfb-best-head-evaluation/0,False,214 +sweeps,ecom-offers,all_four_combined,0,AUROC,roc-auc,higher,0.6505096994576862,0.6505096994576862,0.6063775930816054,greedy-heads,exp/integrated/sweeps/ecom-offers/all_four_combined-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/all_four_combined-greedy-heads-evaluation/0,False,321 +sweeps,ecom-offers,all_minus_cf_fisd,0,AUROC,roc-auc,higher,0.6498655113216565,0.6498655113216565,0.6047694345185003,best-head,exp/integrated/sweeps/ecom-offers/all_minus_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/all_minus_cf_fisd-best-head-evaluation/0,False,321 +sweeps,ecom-offers,all_minus_cf_fisd,0,AUROC,roc-auc,higher,0.6505072081895442,0.6505072081895442,0.6062618182365493,greedy-heads,exp/integrated/sweeps/ecom-offers/all_minus_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/all_minus_cf_fisd-greedy-heads-evaluation/0,False,321 +sweeps,ecom-offers,all_minus_mfb,0,AUROC,roc-auc,higher,0.6386704340734016,0.6386704340734016,0.6021744792778928,greedy-heads,exp/integrated/sweeps/ecom-offers/all_minus_mfb-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/all_minus_mfb-greedy-heads-evaluation/0,False,214 +sweeps,homesite-insurance,rla_rank2_noise0.001,0,AUROC,roc-auc,higher,0.958447068787368,0.958447068787368,0.9625908587761747,mean,exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.001-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.001-evaluation/0,False,1980 +sweeps,homesite-insurance,rla_rank2_noise0.0001,0,AUROC,roc-auc,higher,0.9582141658213201,0.9582141658213201,0.962170313614336,mean,exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.0001-evaluation/0,False,1760 +sweeps,homesite-insurance,rla_rank2_noise1e-05,0,AUROC,roc-auc,higher,0.9583854185194205,0.9583854185194205,0.9623873274822505,mean,exp/integrated/sweeps/homesite-insurance/rla_rank2_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_rank2_noise1e-05-evaluation/0,False,1760 +sweeps,homesite-insurance,rla_rank4_noise0.001,0,AUROC,roc-auc,higher,0.9581143050478821,0.9581143050478821,0.9619551240015043,mean,exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.001-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.001-evaluation/0,False,1100 +sweeps,homesite-insurance,rla_rank4_noise0.0001,0,AUROC,roc-auc,higher,0.9583231528038466,0.9583231528038466,0.96242593886798,mean,exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.0001-evaluation/0,False,1980 +sweeps,homesite-insurance,rla_rank4_noise1e-05,0,AUROC,roc-auc,higher,0.9581626616470993,0.9581626616470993,0.9620964494052155,mean,exp/integrated/sweeps/homesite-insurance/rla_rank4_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_rank4_noise1e-05-evaluation/0,False,1760 +sweeps,homesite-insurance,rla_rank8_noise0.001,0,AUROC,roc-auc,higher,0.9580761824634811,0.9580761824634811,0.962025146401907,mean,exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.001-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.001-evaluation/0,False,1760 +sweeps,homesite-insurance,rla_rank8_noise0.0001,0,AUROC,roc-auc,higher,0.9582609925935802,0.9582609925935802,0.9623517001429152,mean,exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.0001-evaluation/0,False,1100 +sweeps,homesite-insurance,rla_rank8_noise1e-05,0,AUROC,roc-auc,higher,0.958187182838957,0.958187182838957,0.9624786972914716,mean,exp/integrated/sweeps/homesite-insurance/rla_rank8_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_rank8_noise1e-05-evaluation/0,False,1760 +sweeps,homesite-insurance,esam_rho0.001,0,AUROC,roc-auc,higher,0.958366515485181,0.958366515485181,0.9625169220800972,mean,exp/integrated/sweeps/homesite-insurance/esam_rho0.001-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/esam_rho0.001-evaluation/0,False,1980 +sweeps,homesite-insurance,esam_rho0.0025,0,AUROC,roc-auc,higher,0.9581563313286563,0.9581563313286563,0.96205585670933,mean,exp/integrated/sweeps/homesite-insurance/esam_rho0.0025-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/esam_rho0.0025-evaluation/0,False,1760 +sweeps,homesite-insurance,esam_rho0.005,0,AUROC,roc-auc,higher,0.9583120395781356,0.9583120395781356,0.9627483366901237,mean,exp/integrated/sweeps/homesite-insurance/esam_rho0.005-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/esam_rho0.005-evaluation/0,False,1980 +sweeps,homesite-insurance,esam_rho0.01,0,AUROC,roc-auc,higher,0.9580370399944422,0.9580370399944422,0.9621106810110944,mean,exp/integrated/sweeps/homesite-insurance/esam_rho0.01-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/esam_rho0.01-evaluation/0,False,1980 +sweeps,homesite-insurance,mfb_keep0.7,0,AUROC,roc-auc,higher,0.9574455596570353,0.9574455596570353,0.9607387445394369,mean,exp/integrated/sweeps/homesite-insurance/mfb_keep0.7-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/mfb_keep0.7-evaluation/0,False,1980 +sweeps,homesite-insurance,mfb_keep0.8,0,AUROC,roc-auc,higher,0.9580661506671708,0.9580661506671708,0.9619981087669689,mean,exp/integrated/sweeps/homesite-insurance/mfb_keep0.8-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/mfb_keep0.8-evaluation/0,False,1760 +sweeps,homesite-insurance,mfb_keep0.9,0,AUROC,roc-auc,higher,0.9580942942079151,0.9580942942079151,0.9616652728230258,mean,exp/integrated/sweeps/homesite-insurance/mfb_keep0.9-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/mfb_keep0.9-evaluation/0,False,1980 +sweeps,homesite-insurance,mfb_keep0.95,0,AUROC,roc-auc,higher,0.9584246137411132,0.9584246137411132,0.962318960200698,mean,exp/integrated/sweeps/homesite-insurance/mfb_keep0.95-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/mfb_keep0.95-evaluation/0,False,1980 +sweeps,homesite-insurance,cf_fisd_lambda0.05,0,AUROC,roc-auc,higher,0.9587449454385457,0.9587449454385457,0.9627875642149869,mean,exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.05-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.05-evaluation/0,False,1980 +sweeps,homesite-insurance,cf_fisd_lambda0.1,0,AUROC,roc-auc,higher,0.9589599740471012,0.9589599740471012,0.9628908822909436,mean,exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.1-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.1-evaluation/0,False,2420 +sweeps,homesite-insurance,cf_fisd_lambda0.2,0,AUROC,roc-auc,higher,0.9587151577734279,0.9587151577734279,0.9626366101271623,mean,exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.2-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.2-evaluation/0,False,2200 +sweeps,homesite-insurance,rla_esam,0,AUROC,roc-auc,higher,0.9581143402163179,0.9581143402163179,0.962225790298713,mean,exp/integrated/sweeps/homesite-insurance/rla_esam-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_esam-evaluation/0,False,1760 +sweeps,homesite-insurance,rla_mfb,0,AUROC,roc-auc,higher,0.9581138918187615,0.9581138918187615,0.9616453389098677,mean,exp/integrated/sweeps/homesite-insurance/rla_mfb-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_mfb-evaluation/0,False,1980 +sweeps,homesite-insurance,rla_cf_fisd,0,AUROC,roc-auc,higher,0.9584762673811861,0.9584762673811861,0.9621948866927381,mean,exp/integrated/sweeps/homesite-insurance/rla_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_cf_fisd-evaluation/0,False,1100 +sweeps,homesite-insurance,esam_mfb,0,AUROC,roc-auc,higher,0.9580893002900325,0.9580893002900325,0.9618081204528327,mean,exp/integrated/sweeps/homesite-insurance/esam_mfb-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/esam_mfb-evaluation/0,False,1980 +sweeps,homesite-insurance,esam_cf_fisd,0,AUROC,roc-auc,higher,0.9589333075806602,0.9589333075806602,0.9627584002959787,mean,exp/integrated/sweeps/homesite-insurance/esam_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/esam_cf_fisd-evaluation/0,False,2200 +sweeps,homesite-insurance,mfb_cf_fisd,0,AUROC,roc-auc,higher,0.9589238560635404,0.9589238560635404,0.9624954901031625,mean,exp/integrated/sweeps/homesite-insurance/mfb_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/mfb_cf_fisd-evaluation/0,False,1980 +sweeps,homesite-insurance,all_four_combined,0,AUROC,roc-auc,higher,0.958474253988237,0.958474253988237,0.962487069534998,mean,exp/integrated/sweeps/homesite-insurance/all_four_combined-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/all_four_combined-evaluation/0,False,1760 +sweeps,homesite-insurance,all_minus_rla,0,AUROC,roc-auc,higher,0.9586943908120914,0.9586943908120914,0.9623596978704976,mean,exp/integrated/sweeps/homesite-insurance/all_minus_rla-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/all_minus_rla-evaluation/0,False,1980 +sweeps,homesite-insurance,all_minus_esam,0,AUROC,roc-auc,higher,0.95846243739381,0.95846243739381,0.9621719566520266,mean,exp/integrated/sweeps/homesite-insurance/all_minus_esam-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/all_minus_esam-evaluation/0,False,1100 +sweeps,homesite-insurance,all_minus_mfb,0,AUROC,roc-auc,higher,0.9585091762449806,0.9585091762449806,0.9622794548091665,mean,exp/integrated/sweeps/homesite-insurance/all_minus_mfb-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/all_minus_mfb-evaluation/0,False,1100 +sweeps,homesite-insurance,all_minus_cf_fisd,0,AUROC,roc-auc,higher,0.9580700807398709,0.9580700807398709,0.9619997518046595,mean,exp/integrated/sweeps/homesite-insurance/all_minus_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/all_minus_cf_fisd-evaluation/0,False,1980 +sweeps,homesite-insurance,rla_rank2_noise0.0001,0,AUROC,roc-auc,higher,0.9559529233208351,0.9559529233208351,0.9601378276664159,best-head,exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.0001-best-head-evaluation/0,False,1760 +sweeps,homesite-insurance,rla_rank2_noise1e-05,0,AUROC,roc-auc,higher,0.9558571420859494,0.9558571420859494,0.958774227194806,best-head,exp/integrated/sweeps/homesite-insurance/rla_rank2_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_rank2_noise1e-05-best-head-evaluation/0,False,1760 +sweeps,homesite-insurance,rla_rank2_noise0.001,0,AUROC,roc-auc,higher,0.9553321388843028,0.9553321388843028,0.9596838539362689,best-head,exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.001-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.001-best-head-evaluation/0,False,1980 +sweeps,homesite-insurance,rla_rank2_noise0.001,0,AUROC,roc-auc,higher,0.9591161746546817,0.9591161746546817,0.9623163265079293,greedy-heads,exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.001-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.001-greedy-heads-evaluation/0,False,1980 +sweeps,homesite-insurance,rla_rank2_noise0.0001,0,AUROC,roc-auc,higher,0.9589703663198782,0.9589703663198782,0.9625895781732687,greedy-heads,exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.0001-greedy-heads-evaluation/0,False,1760 +sweeps,homesite-insurance,rla_rank2_noise1e-05,0,AUROC,roc-auc,higher,0.9591292133522524,0.9591292133522524,0.9626703044809795,greedy-heads,exp/integrated/sweeps/homesite-insurance/rla_rank2_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_rank2_noise1e-05-greedy-heads-evaluation/0,False,1760 +sweeps,homesite-insurance,rla_rank4_noise0.001,0,AUROC,roc-auc,higher,0.9576809947504603,0.9576809947504603,0.9611084763444616,best-head,exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.001-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.001-best-head-evaluation/0,False,1100 +sweeps,homesite-insurance,rla_rank4_noise0.001,0,AUROC,roc-auc,higher,0.9589014449778305,0.9589014449778305,0.9626523760402965,greedy-heads,exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.001-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.001-greedy-heads-evaluation/0,False,1100 +sweeps,homesite-insurance,esam_rho0.0025,0,AUROC,roc-auc,higher,0.9561005955827354,0.9561005955827354,0.9607496659076156,best-head,exp/integrated/sweeps/homesite-insurance/esam_rho0.0025-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/esam_rho0.0025-best-head-evaluation/0,False,1760 +sweeps,homesite-insurance,mfb_keep0.7,0,AUROC,roc-auc,higher,0.9540607823461205,0.9540607823461205,0.9604495578247303,best-head,exp/integrated/sweeps/homesite-insurance/mfb_keep0.7-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/mfb_keep0.7-best-head-evaluation/0,False,1980 +sweeps,homesite-insurance,esam_rho0.001,0,AUROC,roc-auc,higher,0.9561281764285071,0.9561281764285071,0.9607344074031798,best-head,exp/integrated/sweeps/homesite-insurance/esam_rho0.001-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/esam_rho0.001-best-head-evaluation/0,False,1980 +sweeps,homesite-insurance,mfb_keep0.8,0,AUROC,roc-auc,higher,0.9557633918282306,0.9557633918282306,0.9594055402651032,best-head,exp/integrated/sweeps/homesite-insurance/mfb_keep0.8-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/mfb_keep0.8-best-head-evaluation/0,False,1760 +sweeps,homesite-insurance,mfb_keep0.7,0,AUROC,roc-auc,higher,0.9588480241238587,0.9588480241238587,0.9630162122394903,greedy-heads,exp/integrated/sweeps/homesite-insurance/mfb_keep0.7-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/mfb_keep0.7-greedy-heads-evaluation/0,False,1980 +sweeps,homesite-insurance,esam_rho0.0025,0,AUROC,roc-auc,higher,0.95870368407125,0.95870368407125,0.962465432178352,greedy-heads,exp/integrated/sweeps/homesite-insurance/esam_rho0.0025-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/esam_rho0.0025-greedy-heads-evaluation/0,False,1760 +sweeps,homesite-insurance,mfb_keep0.9,0,AUROC,roc-auc,higher,0.9560981250001208,0.9560981250001208,0.9601610718172741,best-head,exp/integrated/sweeps/homesite-insurance/mfb_keep0.9-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/mfb_keep0.9-best-head-evaluation/0,False,1980 +sweeps,homesite-insurance,esam_rho0.001,0,AUROC,roc-auc,higher,0.9589766350935588,0.9589766350935588,0.962915757398333,greedy-heads,exp/integrated/sweeps/homesite-insurance/esam_rho0.001-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/esam_rho0.001-greedy-heads-evaluation/0,False,1980 +sweeps,homesite-insurance,mfb_keep0.8,0,AUROC,roc-auc,higher,0.9587700557017027,0.9587700557017027,0.9623207965369405,greedy-heads,exp/integrated/sweeps/homesite-insurance/mfb_keep0.8-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/mfb_keep0.8-greedy-heads-evaluation/0,False,1760 +sweeps,homesite-insurance,esam_rho0.005,0,AUROC,roc-auc,higher,0.9561512908829329,0.9561512908829329,0.9606910602029306,best-head,exp/integrated/sweeps/homesite-insurance/esam_rho0.005-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/esam_rho0.005-best-head-evaluation/0,False,1980 +sweeps,homesite-insurance,mfb_keep0.9,0,AUROC,roc-auc,higher,0.9587921238951639,0.9587921238951639,0.9622146031450255,greedy-heads,exp/integrated/sweeps/homesite-insurance/mfb_keep0.9-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/mfb_keep0.9-greedy-heads-evaluation/0,False,1980 +sweeps,homesite-insurance,esam_rho0.01,0,AUROC,roc-auc,higher,0.9554589123032324,0.9554589123032324,0.9585579261153012,best-head,exp/integrated/sweeps/homesite-insurance/esam_rho0.01-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/esam_rho0.01-best-head-evaluation/0,False,1980 +sweeps,homesite-insurance,esam_rho0.005,0,AUROC,roc-auc,higher,0.958991133281215,0.958991133281215,0.9627938464179217,greedy-heads,exp/integrated/sweeps/homesite-insurance/esam_rho0.005-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/esam_rho0.005-greedy-heads-evaluation/0,False,1980 +sweeps,homesite-insurance,esam_rho0.01,0,AUROC,roc-auc,higher,0.9587733087820138,0.9587733087820138,0.9626104060922291,greedy-heads,exp/integrated/sweeps/homesite-insurance/esam_rho0.01-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/esam_rho0.01-greedy-heads-evaluation/0,False,1980 +sweeps,homesite-insurance,rla_rank4_noise0.0001,0,AUROC,roc-auc,higher,0.9557610707114682,0.9557610707114682,0.9604946447119459,best-head,exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.0001-best-head-evaluation/0,False,1980 +sweeps,homesite-insurance,rla_rank4_noise0.0001,0,AUROC,roc-auc,higher,0.959029757015826,0.959029757015826,0.9624579176971494,greedy-heads,exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.0001-greedy-heads-evaluation/0,False,1980 +sweeps,homesite-insurance,mfb_keep0.95,0,AUROC,roc-auc,higher,0.9563975490624737,0.9563975490624737,0.9593891219693567,best-head,exp/integrated/sweeps/homesite-insurance/mfb_keep0.95-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/mfb_keep0.95-best-head-evaluation/0,False,1980 +sweeps,homesite-insurance,rla_rank4_noise1e-05,0,AUROC,roc-auc,higher,0.9560453459701026,0.9560453459701026,0.9600924508313724,best-head,exp/integrated/sweeps/homesite-insurance/rla_rank4_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_rank4_noise1e-05-best-head-evaluation/0,False,1760 +sweeps,homesite-insurance,mfb_keep0.95,0,AUROC,roc-auc,higher,0.9592957095194116,0.9592957095194116,0.9620218603265258,greedy-heads,exp/integrated/sweeps/homesite-insurance/mfb_keep0.95-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/mfb_keep0.95-greedy-heads-evaluation/0,False,1980 +sweeps,homesite-insurance,rla_rank4_noise1e-05,0,AUROC,roc-auc,higher,0.9591329763748824,0.9591329763748824,0.962370812537228,greedy-heads,exp/integrated/sweeps/homesite-insurance/rla_rank4_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_rank4_noise1e-05-greedy-heads-evaluation/0,False,1760 +sweeps,homesite-insurance,cf_fisd_lambda0.05,0,AUROC,roc-auc,higher,0.9577627701557911,0.9577627701557911,0.9622852537657216,best-head,exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.05-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.05-best-head-evaluation/0,False,1980 +sweeps,homesite-insurance,cf_fisd_lambda0.05,0,AUROC,roc-auc,higher,0.9594017599375494,0.9594017599375494,0.9632589952205967,greedy-heads,exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.05-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.05-greedy-heads-evaluation/0,False,1980 +sweeps,homesite-insurance,cf_fisd_lambda0.1,0,AUROC,roc-auc,higher,0.957813544584969,0.957813544584969,0.961490313471295,best-head,exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.1-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.1-best-head-evaluation/0,False,2420 +sweeps,homesite-insurance,cf_fisd_lambda0.1,0,AUROC,roc-auc,higher,0.9594844321379927,0.9594844321379927,0.9634382071404677,greedy-heads,exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.1-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.1-greedy-heads-evaluation/0,False,2420 +sweeps,homesite-insurance,cf_fisd_lambda0.2,0,AUROC,roc-auc,higher,0.9576897428988642,0.9576897428988642,0.9612223171103336,best-head,exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.2-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.2-best-head-evaluation/0,False,2200 +sweeps,homesite-insurance,cf_fisd_lambda0.2,0,AUROC,roc-auc,higher,0.9594008895187635,0.9594008895187635,0.9626779880984148,greedy-heads,exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.2-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.2-greedy-heads-evaluation/0,False,2200 +sweeps,homesite-insurance,mfb_cf_fisd,0,AUROC,roc-auc,higher,0.9576967765860231,0.9576967765860231,0.9617315500639867,best-head,exp/integrated/sweeps/homesite-insurance/mfb_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/mfb_cf_fisd-best-head-evaluation/0,False,1980 +sweeps,homesite-insurance,mfb_cf_fisd,0,AUROC,roc-auc,higher,0.9597570226838343,0.9597570226838343,0.9633062567165207,greedy-heads,exp/integrated/sweeps/homesite-insurance/mfb_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/mfb_cf_fisd-greedy-heads-evaluation/0,False,1980 +sweeps,homesite-insurance,rla_rank8_noise0.0001,0,AUROC,roc-auc,higher,0.9580729733437149,0.9580729733437149,0.9613215034297445,best-head,exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.0001-best-head-evaluation/0,False,1100 +sweeps,homesite-insurance,rla_rank8_noise0.0001,0,AUROC,roc-auc,higher,0.9588554974164651,0.9588554974164651,0.9627966975715613,greedy-heads,exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.0001-greedy-heads-evaluation/0,False,1100 +sweeps,homesite-insurance,esam_mfb,0,AUROC,roc-auc,higher,0.9551748656394307,0.9551748656394307,0.9597148541915197,best-head,exp/integrated/sweeps/homesite-insurance/esam_mfb-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/esam_mfb-best-head-evaluation/0,False,1980 +sweeps,homesite-insurance,esam_mfb,0,AUROC,roc-auc,higher,0.9588471624971818,0.9588471624971818,0.962094141903753,greedy-heads,exp/integrated/sweeps/homesite-insurance/esam_mfb-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/esam_mfb-greedy-heads-evaluation/0,False,1980 +sweeps,homesite-insurance,rla_rank8_noise0.001,0,AUROC,roc-auc,higher,0.9565663135937417,0.9565663135937417,0.9586702567362371,best-head,exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.001-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.001-best-head-evaluation/0,False,1760 +sweeps,homesite-insurance,rla_rank8_noise0.001,0,AUROC,roc-auc,higher,0.9588770556676071,0.9588770556676071,0.9621469244895637,greedy-heads,exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.001-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.001-greedy-heads-evaluation/0,False,1760 +sweeps,homesite-insurance,rla_rank8_noise1e-05,0,AUROC,roc-auc,higher,0.9560226359526883,0.9560226359526883,0.9594298354768371,best-head,exp/integrated/sweeps/homesite-insurance/rla_rank8_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_rank8_noise1e-05-best-head-evaluation/0,False,1760 +sweeps,homesite-insurance,rla_rank8_noise1e-05,0,AUROC,roc-auc,higher,0.9588073342436448,0.9588073342436448,0.9625660923992208,greedy-heads,exp/integrated/sweeps/homesite-insurance/rla_rank8_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_rank8_noise1e-05-greedy-heads-evaluation/0,False,1760 +sweeps,homesite-insurance,rla_cf_fisd,0,AUROC,roc-auc,higher,0.9580215219221477,0.9580215219221477,0.9612981867919294,best-head,exp/integrated/sweeps/homesite-insurance/rla_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_cf_fisd-best-head-evaluation/0,False,1100 +sweeps,homesite-insurance,esam_cf_fisd,0,AUROC,roc-auc,higher,0.9573917959108148,0.9573917959108148,0.9606087512633268,best-head,exp/integrated/sweeps/homesite-insurance/esam_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/esam_cf_fisd-best-head-evaluation/0,False,2200 +sweeps,homesite-insurance,rla_cf_fisd,0,AUROC,roc-auc,higher,0.9589731006657612,0.9589731006657612,0.9623105275513741,greedy-heads,exp/integrated/sweeps/homesite-insurance/rla_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_cf_fisd-greedy-heads-evaluation/0,False,1100 +sweeps,homesite-insurance,esam_cf_fisd,0,AUROC,roc-auc,higher,0.9595481045909983,0.9595481045909983,0.9631976229303886,greedy-heads,exp/integrated/sweeps/homesite-insurance/esam_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/esam_cf_fisd-greedy-heads-evaluation/0,False,2200 +sweeps,homesite-insurance,rla_mfb,0,AUROC,roc-auc,higher,0.9554584990741118,0.9554584990741118,0.9598635611836792,best-head,exp/integrated/sweeps/homesite-insurance/rla_mfb-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_mfb-best-head-evaluation/0,False,1980 +sweeps,homesite-insurance,all_minus_rla,0,AUROC,roc-auc,higher,0.9573359044742286,0.9573359044742286,0.9616503767533748,best-head,exp/integrated/sweeps/homesite-insurance/all_minus_rla-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/all_minus_rla-best-head-evaluation/0,False,1980 +sweeps,homesite-insurance,rla_mfb,0,AUROC,roc-auc,higher,0.9589065795694565,0.9589065795694565,0.961628401124263,greedy-heads,exp/integrated/sweeps/homesite-insurance/rla_mfb-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_mfb-greedy-heads-evaluation/0,False,1980 +sweeps,homesite-insurance,all_minus_rla,0,AUROC,roc-auc,higher,0.9594920636885601,0.9594920636885601,0.9630834801355295,greedy-heads,exp/integrated/sweeps/homesite-insurance/all_minus_rla-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/all_minus_rla-greedy-heads-evaluation/0,False,1980 +sweeps,homesite-insurance,all_minus_esam,0,AUROC,roc-auc,higher,0.9578664730808395,0.9578664730808395,0.9618357621457454,best-head,exp/integrated/sweeps/homesite-insurance/all_minus_esam-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/all_minus_esam-best-head-evaluation/0,False,1100 +sweeps,homesite-insurance,all_minus_esam,0,AUROC,roc-auc,higher,0.959329480009883,0.959329480009883,0.9630054358452254,greedy-heads,exp/integrated/sweeps/homesite-insurance/all_minus_esam-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/all_minus_esam-greedy-heads-evaluation/0,False,1100 +sweeps,homesite-insurance,rla_esam,0,AUROC,roc-auc,higher,0.9561241320583908,0.9561241320583908,0.960971584726283,best-head,exp/integrated/sweeps/homesite-insurance/rla_esam-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_esam-best-head-evaluation/0,False,1760 +sweeps,homesite-insurance,rla_esam,0,AUROC,roc-auc,higher,0.9589296236870106,0.9589296236870106,0.9624745655349262,greedy-heads,exp/integrated/sweeps/homesite-insurance/rla_esam-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_esam-greedy-heads-evaluation/0,False,1760 +sweeps,homesite-insurance,all_minus_mfb,0,AUROC,roc-auc,higher,0.9578946605821286,0.9578946605821286,0.9613487585255535,best-head,exp/integrated/sweeps/homesite-insurance/all_minus_mfb-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/all_minus_mfb-best-head-evaluation/0,False,1100 +sweeps,homesite-insurance,all_minus_mfb,0,AUROC,roc-auc,higher,0.9590512800985321,0.9590512800985321,0.9625182147641627,greedy-heads,exp/integrated/sweeps/homesite-insurance/all_minus_mfb-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/all_minus_mfb-greedy-heads-evaluation/0,False,1100 +sweeps,homesite-insurance,all_four_combined,0,AUROC,roc-auc,higher,0.9569685349939213,0.9569685349939213,0.9582203301945192,best-head,exp/integrated/sweeps/homesite-insurance/all_four_combined-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/all_four_combined-best-head-evaluation/0,False,1760 +sweeps,homesite-insurance,all_four_combined,0,AUROC,roc-auc,higher,0.9592737644154757,0.9592737644154757,0.9627526496640615,greedy-heads,exp/integrated/sweeps/homesite-insurance/all_four_combined-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/all_four_combined-greedy-heads-evaluation/0,False,1760 +sweeps,homesite-insurance,all_minus_cf_fisd,0,AUROC,roc-auc,higher,0.9560637742304589,0.9560637742304589,0.9602624689888717,best-head,exp/integrated/sweeps/homesite-insurance/all_minus_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/all_minus_cf_fisd-best-head-evaluation/0,False,1980 +sweeps,homesite-insurance,all_minus_cf_fisd,0,AUROC,roc-auc,higher,0.9587104803714672,0.9587104803714672,0.9619912225060596,greedy-heads,exp/integrated/sweeps/homesite-insurance/all_minus_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/all_minus_cf_fisd-greedy-heads-evaluation/0,False,1980 +sweeps,cooking-time,rla_rank2_noise0.001,0,RMSE,rmse,lower,0.462741360963445,-0.462741360963445,0.4802330803717953,mean,exp/integrated/sweeps/cooking-time/rla_rank2_noise0.001-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_rank2_noise0.001-evaluation/0,False,23088 +sweeps,cooking-time,rla_rank2_noise0.0001,0,RMSE,rmse,lower,0.4627581378606596,-0.4627581378606596,0.4802975698569232,mean,exp/integrated/sweeps/cooking-time/rla_rank2_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_rank2_noise0.0001-evaluation/0,False,23088 +sweeps,cooking-time,rla_rank2_noise1e-05,0,RMSE,rmse,lower,0.46274171518445095,-0.46274171518445095,0.48025462954174786,mean,exp/integrated/sweeps/cooking-time/rla_rank2_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_rank2_noise1e-05-evaluation/0,False,23088 +sweeps,cooking-time,rla_rank4_noise0.001,0,RMSE,rmse,lower,0.46272482498106277,-0.46272482498106277,0.4802920784266626,mean,exp/integrated/sweeps/cooking-time/rla_rank4_noise0.001-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_rank4_noise0.001-evaluation/0,False,23088 +sweeps,cooking-time,rla_rank4_noise0.0001,0,RMSE,rmse,lower,0.4627297681263241,-0.4627297681263241,0.48025858554781287,mean,exp/integrated/sweeps/cooking-time/rla_rank4_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_rank4_noise0.0001-evaluation/0,False,23088 +sweeps,cooking-time,rla_rank4_noise1e-05,0,RMSE,rmse,lower,0.4627757191386438,-0.4627757191386438,0.4802501150008962,mean,exp/integrated/sweeps/cooking-time/rla_rank4_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_rank4_noise1e-05-evaluation/0,False,23088 +sweeps,cooking-time,rla_rank8_noise0.001,0,RMSE,rmse,lower,0.4627485258356125,-0.4627485258356125,0.4803784913673949,mean,exp/integrated/sweeps/cooking-time/rla_rank8_noise0.001-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_rank8_noise0.001-evaluation/0,False,23088 +sweeps,cooking-time,rla_rank8_noise0.0001,0,RMSE,rmse,lower,0.4627781501968618,-0.4627781501968618,0.4803483238337491,mean,exp/integrated/sweeps/cooking-time/rla_rank8_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_rank8_noise0.0001-evaluation/0,False,23088 +sweeps,cooking-time,rla_rank8_noise1e-05,0,RMSE,rmse,lower,0.46274024999580365,-0.46274024999580365,0.48030301468765013,mean,exp/integrated/sweeps/cooking-time/rla_rank8_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_rank8_noise1e-05-evaluation/0,False,23088 +sweeps,cooking-time,esam_rho0.001,0,RMSE,rmse,lower,0.46276101981921397,-0.46276101981921397,0.4801992110036178,mean,exp/integrated/sweeps/cooking-time/esam_rho0.001-evaluation/0.toml,exp/integrated/sweeps/cooking-time/esam_rho0.001-evaluation/0,False,23088 +sweeps,cooking-time,esam_rho0.0025,0,RMSE,rmse,lower,0.4627563346148938,-0.4627563346148938,0.4802735094812688,mean,exp/integrated/sweeps/cooking-time/esam_rho0.0025-evaluation/0.toml,exp/integrated/sweeps/cooking-time/esam_rho0.0025-evaluation/0,False,23088 +sweeps,cooking-time,esam_rho0.005,0,RMSE,rmse,lower,0.4628077726618535,-0.4628077726618535,0.48022789849772834,mean,exp/integrated/sweeps/cooking-time/esam_rho0.005-evaluation/0.toml,exp/integrated/sweeps/cooking-time/esam_rho0.005-evaluation/0,False,23088 +sweeps,cooking-time,esam_rho0.01,0,RMSE,rmse,lower,0.462806194991412,-0.462806194991412,0.4802449022823802,mean,exp/integrated/sweeps/cooking-time/esam_rho0.01-evaluation/0.toml,exp/integrated/sweeps/cooking-time/esam_rho0.01-evaluation/0,False,23088 +sweeps,cooking-time,mfb_keep0.7,0,RMSE,rmse,lower,0.4641162243965573,-0.4641162243965573,0.4815966695140456,mean,exp/integrated/sweeps/cooking-time/mfb_keep0.7-evaluation/0.toml,exp/integrated/sweeps/cooking-time/mfb_keep0.7-evaluation/0,False,23088 +sweeps,cooking-time,mfb_keep0.8,0,RMSE,rmse,lower,0.4634085546239816,-0.4634085546239816,0.4808915146871742,mean,exp/integrated/sweeps/cooking-time/mfb_keep0.8-evaluation/0.toml,exp/integrated/sweeps/cooking-time/mfb_keep0.8-evaluation/0,False,23088 +sweeps,cooking-time,mfb_keep0.9,0,RMSE,rmse,lower,0.46290981075910853,-0.46290981075910853,0.480415480847424,mean,exp/integrated/sweeps/cooking-time/mfb_keep0.9-evaluation/0.toml,exp/integrated/sweeps/cooking-time/mfb_keep0.9-evaluation/0,False,23088 +sweeps,cooking-time,mfb_keep0.95,0,RMSE,rmse,lower,0.46283832689198484,-0.46283832689198484,0.48036774294670015,mean,exp/integrated/sweeps/cooking-time/mfb_keep0.95-evaluation/0.toml,exp/integrated/sweeps/cooking-time/mfb_keep0.95-evaluation/0,False,23088 +sweeps,cooking-time,cf_fisd_lambda0.05,0,RMSE,rmse,lower,0.4632019090821483,-0.4632019090821483,0.4808496810468957,mean,exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.05-evaluation/0.toml,exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.05-evaluation/0,False,17982 +sweeps,cooking-time,cf_fisd_lambda0.1,0,RMSE,rmse,lower,0.4632809116304394,-0.4632809116304394,0.48091704691326154,mean,exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.1-evaluation/0.toml,exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.1-evaluation/0,False,17982 +sweeps,cooking-time,cf_fisd_lambda0.2,0,RMSE,rmse,lower,0.4633341406854049,-0.4633341406854049,0.4808806847716492,mean,exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.2-evaluation/0.toml,exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.2-evaluation/0,False,15096 +sweeps,cooking-time,rla_esam,0,RMSE,rmse,lower,0.4627353230636857,-0.4627353230636857,0.4803077303885999,mean,exp/integrated/sweeps/cooking-time/rla_esam-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_esam-evaluation/0,False,23088 +sweeps,cooking-time,rla_mfb,0,RMSE,rmse,lower,0.4628964355361592,-0.4628964355361592,0.4804232040775468,mean,exp/integrated/sweeps/cooking-time/rla_mfb-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_mfb-evaluation/0,False,23088 +sweeps,cooking-time,rla_cf_fisd,0,RMSE,rmse,lower,0.46314488441622087,-0.46314488441622087,0.48088551875549623,mean,exp/integrated/sweeps/cooking-time/rla_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_cf_fisd-evaluation/0,False,17982 +sweeps,cooking-time,esam_mfb,0,RMSE,rmse,lower,0.4629259699594227,-0.4629259699594227,0.4803877661460669,mean,exp/integrated/sweeps/cooking-time/esam_mfb-evaluation/0.toml,exp/integrated/sweeps/cooking-time/esam_mfb-evaluation/0,False,23088 +sweeps,cooking-time,esam_cf_fisd,0,RMSE,rmse,lower,0.4632865725335359,-0.4632865725335359,0.48093585437297304,mean,exp/integrated/sweeps/cooking-time/esam_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/cooking-time/esam_cf_fisd-evaluation/0,False,17982 +sweeps,cooking-time,mfb_cf_fisd,0,RMSE,rmse,lower,0.4633989882463895,-0.4633989882463895,0.4810100390661497,mean,exp/integrated/sweeps/cooking-time/mfb_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/cooking-time/mfb_cf_fisd-evaluation/0,False,17982 +sweeps,cooking-time,all_four_combined,0,RMSE,rmse,lower,0.463355012524859,-0.463355012524859,0.4809202538385541,mean,exp/integrated/sweeps/cooking-time/all_four_combined-evaluation/0.toml,exp/integrated/sweeps/cooking-time/all_four_combined-evaluation/0,False,17982 +sweeps,cooking-time,all_minus_rla,0,RMSE,rmse,lower,0.4634391174751191,-0.4634391174751191,0.4810216869909678,mean,exp/integrated/sweeps/cooking-time/all_minus_rla-evaluation/0.toml,exp/integrated/sweeps/cooking-time/all_minus_rla-evaluation/0,False,17982 +sweeps,cooking-time,all_minus_esam,0,RMSE,rmse,lower,0.4633378873945458,-0.4633378873945458,0.48088029743109706,mean,exp/integrated/sweeps/cooking-time/all_minus_esam-evaluation/0.toml,exp/integrated/sweeps/cooking-time/all_minus_esam-evaluation/0,False,17982 +sweeps,cooking-time,all_minus_mfb,0,RMSE,rmse,lower,0.46319753395446966,-0.46319753395446966,0.48088071575887986,mean,exp/integrated/sweeps/cooking-time/all_minus_mfb-evaluation/0.toml,exp/integrated/sweeps/cooking-time/all_minus_mfb-evaluation/0,False,17982 +sweeps,cooking-time,all_minus_cf_fisd,0,RMSE,rmse,lower,0.463067499804912,-0.463067499804912,0.4804862724768644,mean,exp/integrated/sweeps/cooking-time/all_minus_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/cooking-time/all_minus_cf_fisd-evaluation/0,False,17982 +sweeps,cooking-time,rla_rank2_noise0.001,0,RMSE,rmse,lower,0.465139054815724,-0.465139054815724,0.48220568612119014,best-head,exp/integrated/sweeps/cooking-time/rla_rank2_noise0.001-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_rank2_noise0.001-best-head-evaluation/0,False,23088 +sweeps,cooking-time,rla_rank2_noise0.001,0,RMSE,rmse,lower,0.4624384832710735,-0.4624384832710735,0.47998788853107577,greedy-heads,exp/integrated/sweeps/cooking-time/rla_rank2_noise0.001-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_rank2_noise0.001-greedy-heads-evaluation/0,False,23088 +sweeps,cooking-time,rla_rank2_noise0.0001,0,RMSE,rmse,lower,0.465058894016931,-0.465058894016931,0.48247512338182996,best-head,exp/integrated/sweeps/cooking-time/rla_rank2_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_rank2_noise0.0001-best-head-evaluation/0,False,23088 +sweeps,cooking-time,rla_rank2_noise0.0001,0,RMSE,rmse,lower,0.4624788086134533,-0.4624788086134533,0.4800547545217561,greedy-heads,exp/integrated/sweeps/cooking-time/rla_rank2_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_rank2_noise0.0001-greedy-heads-evaluation/0,False,23088 +sweeps,cooking-time,rla_rank2_noise1e-05,0,RMSE,rmse,lower,0.4650147388029769,-0.4650147388029769,0.4822676099238603,best-head,exp/integrated/sweeps/cooking-time/rla_rank2_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_rank2_noise1e-05-best-head-evaluation/0,False,23088 +sweeps,cooking-time,rla_rank2_noise1e-05,0,RMSE,rmse,lower,0.46249997680251603,-0.46249997680251603,0.47995778958671514,greedy-heads,exp/integrated/sweeps/cooking-time/rla_rank2_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_rank2_noise1e-05-greedy-heads-evaluation/0,False,23088 +sweeps,cooking-time,esam_rho0.001,0,RMSE,rmse,lower,0.4650444270773418,-0.4650444270773418,0.48297671022399175,best-head,exp/integrated/sweeps/cooking-time/esam_rho0.001-evaluation/0.toml,exp/integrated/sweeps/cooking-time/esam_rho0.001-best-head-evaluation/0,False,23088 +sweeps,cooking-time,esam_rho0.001,0,RMSE,rmse,lower,0.4625151676835406,-0.4625151676835406,0.479742369480289,greedy-heads,exp/integrated/sweeps/cooking-time/esam_rho0.001-evaluation/0.toml,exp/integrated/sweeps/cooking-time/esam_rho0.001-greedy-heads-evaluation/0,False,23088 +sweeps,cooking-time,esam_rho0.0025,0,RMSE,rmse,lower,0.46513258351328474,-0.46513258351328474,0.48253251942068065,best-head,exp/integrated/sweeps/cooking-time/esam_rho0.0025-evaluation/0.toml,exp/integrated/sweeps/cooking-time/esam_rho0.0025-best-head-evaluation/0,False,23088 +sweeps,cooking-time,esam_rho0.0025,0,RMSE,rmse,lower,0.46246732196836127,-0.46246732196836127,0.4800169922135662,greedy-heads,exp/integrated/sweeps/cooking-time/esam_rho0.0025-evaluation/0.toml,exp/integrated/sweeps/cooking-time/esam_rho0.0025-greedy-heads-evaluation/0,False,23088 +sweeps,cooking-time,esam_rho0.005,0,RMSE,rmse,lower,0.4651283707110853,-0.4651283707110853,0.4823068025931435,best-head,exp/integrated/sweeps/cooking-time/esam_rho0.005-evaluation/0.toml,exp/integrated/sweeps/cooking-time/esam_rho0.005-best-head-evaluation/0,False,23088 +sweeps,cooking-time,esam_rho0.005,0,RMSE,rmse,lower,0.4625398779765968,-0.4625398779765968,0.4797024392153251,greedy-heads,exp/integrated/sweeps/cooking-time/esam_rho0.005-evaluation/0.toml,exp/integrated/sweeps/cooking-time/esam_rho0.005-greedy-heads-evaluation/0,False,23088 +sweeps,cooking-time,mfb_keep0.7,0,RMSE,rmse,lower,0.46477365365967316,-0.46477365365967316,0.48197477680705497,best-head,exp/integrated/sweeps/cooking-time/mfb_keep0.7-evaluation/0.toml,exp/integrated/sweeps/cooking-time/mfb_keep0.7-best-head-evaluation/0,False,23088 +sweeps,cooking-time,mfb_keep0.7,0,RMSE,rmse,lower,0.46249551448708903,-0.46249551448708903,0.47980768580603567,greedy-heads,exp/integrated/sweeps/cooking-time/mfb_keep0.7-evaluation/0.toml,exp/integrated/sweeps/cooking-time/mfb_keep0.7-greedy-heads-evaluation/0,False,23088 +sweeps,cooking-time,mfb_keep0.8,0,RMSE,rmse,lower,0.4652213638497702,-0.4652213638497702,0.48343941871247514,best-head,exp/integrated/sweeps/cooking-time/mfb_keep0.8-evaluation/0.toml,exp/integrated/sweeps/cooking-time/mfb_keep0.8-best-head-evaluation/0,False,23088 +sweeps,cooking-time,mfb_keep0.8,0,RMSE,rmse,lower,0.4624822561623469,-0.4624822561623469,0.4798057758257153,greedy-heads,exp/integrated/sweeps/cooking-time/mfb_keep0.8-evaluation/0.toml,exp/integrated/sweeps/cooking-time/mfb_keep0.8-greedy-heads-evaluation/0,False,23088 +sweeps,cooking-time,mfb_keep0.9,0,RMSE,rmse,lower,0.465403548325863,-0.465403548325863,0.48311284423774803,best-head,exp/integrated/sweeps/cooking-time/mfb_keep0.9-evaluation/0.toml,exp/integrated/sweeps/cooking-time/mfb_keep0.9-best-head-evaluation/0,False,23088 +sweeps,cooking-time,mfb_keep0.9,0,RMSE,rmse,lower,0.4625593198933149,-0.4625593198933149,0.479899744054174,greedy-heads,exp/integrated/sweeps/cooking-time/mfb_keep0.9-evaluation/0.toml,exp/integrated/sweeps/cooking-time/mfb_keep0.9-greedy-heads-evaluation/0,False,23088 +sweeps,cooking-time,rla_rank4_noise0.001,0,RMSE,rmse,lower,0.4649795846661889,-0.4649795846661889,0.4827695666979872,best-head,exp/integrated/sweeps/cooking-time/rla_rank4_noise0.001-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_rank4_noise0.001-best-head-evaluation/0,False,23088 +sweeps,cooking-time,rla_rank4_noise0.001,0,RMSE,rmse,lower,0.4624107384246987,-0.4624107384246987,0.479974942646056,greedy-heads,exp/integrated/sweeps/cooking-time/rla_rank4_noise0.001-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_rank4_noise0.001-greedy-heads-evaluation/0,False,23088 +sweeps,cooking-time,esam_rho0.01,0,RMSE,rmse,lower,0.46502524927765376,-0.46502524927765376,0.48171681453527604,best-head,exp/integrated/sweeps/cooking-time/esam_rho0.01-evaluation/0.toml,exp/integrated/sweeps/cooking-time/esam_rho0.01-best-head-evaluation/0,False,23088 +sweeps,cooking-time,esam_rho0.01,0,RMSE,rmse,lower,0.46260330683144324,-0.46260330683144324,0.47994838230926673,greedy-heads,exp/integrated/sweeps/cooking-time/esam_rho0.01-evaluation/0.toml,exp/integrated/sweeps/cooking-time/esam_rho0.01-greedy-heads-evaluation/0,False,23088 +sweeps,cooking-time,cf_fisd_lambda0.05,0,RMSE,rmse,lower,0.4638214410008231,-0.4638214410008231,0.4819659035779245,best-head,exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.05-evaluation/0.toml,exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.05-best-head-evaluation/0,False,17982 +sweeps,cooking-time,cf_fisd_lambda0.05,0,RMSE,rmse,lower,0.4628482268025946,-0.4628482268025946,0.4803119031248436,greedy-heads,exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.05-evaluation/0.toml,exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.05-greedy-heads-evaluation/0,False,17982 +sweeps,cooking-time,rla_rank4_noise0.0001,0,RMSE,rmse,lower,0.4651183431272534,-0.4651183431272534,0.48213882482447107,best-head,exp/integrated/sweeps/cooking-time/rla_rank4_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_rank4_noise0.0001-best-head-evaluation/0,False,23088 +sweeps,cooking-time,rla_rank4_noise0.0001,0,RMSE,rmse,lower,0.4624375326912171,-0.4624375326912171,0.48004674599435365,greedy-heads,exp/integrated/sweeps/cooking-time/rla_rank4_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_rank4_noise0.0001-greedy-heads-evaluation/0,False,23088 +sweeps,cooking-time,mfb_keep0.95,0,RMSE,rmse,lower,0.4649031464912309,-0.4649031464912309,0.4828450127182003,best-head,exp/integrated/sweeps/cooking-time/mfb_keep0.95-evaluation/0.toml,exp/integrated/sweeps/cooking-time/mfb_keep0.95-best-head-evaluation/0,False,23088 +sweeps,cooking-time,mfb_keep0.95,0,RMSE,rmse,lower,0.46250350473983365,-0.46250350473983365,0.4799452154615134,greedy-heads,exp/integrated/sweeps/cooking-time/mfb_keep0.95-evaluation/0.toml,exp/integrated/sweeps/cooking-time/mfb_keep0.95-greedy-heads-evaluation/0,False,23088 +sweeps,cooking-time,cf_fisd_lambda0.1,0,RMSE,rmse,lower,0.46397138529545745,-0.46397138529545745,0.4813020969034895,best-head,exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.1-evaluation/0.toml,exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.1-best-head-evaluation/0,False,17982 +sweeps,cooking-time,cf_fisd_lambda0.1,0,RMSE,rmse,lower,0.4629530240908241,-0.4629530240908241,0.480628211999089,greedy-heads,exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.1-evaluation/0.toml,exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.1-greedy-heads-evaluation/0,False,17982 +sweeps,cooking-time,rla_rank4_noise1e-05,0,RMSE,rmse,lower,0.465206517587419,-0.465206517587419,0.4827162118838719,best-head,exp/integrated/sweeps/cooking-time/rla_rank4_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_rank4_noise1e-05-best-head-evaluation/0,False,23088 +sweeps,cooking-time,rla_rank4_noise1e-05,0,RMSE,rmse,lower,0.46247985576887213,-0.46247985576887213,0.47992046986293,greedy-heads,exp/integrated/sweeps/cooking-time/rla_rank4_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_rank4_noise1e-05-greedy-heads-evaluation/0,False,23088 +sweeps,cooking-time,cf_fisd_lambda0.2,0,RMSE,rmse,lower,0.4641009735445309,-0.4641009735445309,0.48193517064355434,best-head,exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.2-evaluation/0.toml,exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.2-best-head-evaluation/0,False,15096 +sweeps,cooking-time,cf_fisd_lambda0.2,0,RMSE,rmse,lower,0.463044314080512,-0.463044314080512,0.4804926610439783,greedy-heads,exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.2-evaluation/0.toml,exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.2-greedy-heads-evaluation/0,False,15096 +sweeps,cooking-time,mfb_cf_fisd,0,RMSE,rmse,lower,0.46410829401604725,-0.46410829401604725,0.4815192949196226,best-head,exp/integrated/sweeps/cooking-time/mfb_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/cooking-time/mfb_cf_fisd-best-head-evaluation/0,False,17982 +sweeps,cooking-time,mfb_cf_fisd,0,RMSE,rmse,lower,0.46284785656584665,-0.46284785656584665,0.4803782742299631,greedy-heads,exp/integrated/sweeps/cooking-time/mfb_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/cooking-time/mfb_cf_fisd-greedy-heads-evaluation/0,False,17982 +sweeps,cooking-time,rla_rank8_noise0.001,0,RMSE,rmse,lower,0.4648571653594056,-0.4648571653594056,0.4833629710740515,best-head,exp/integrated/sweeps/cooking-time/rla_rank8_noise0.001-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_rank8_noise0.001-best-head-evaluation/0,False,23088 +sweeps,cooking-time,rla_rank8_noise0.001,0,RMSE,rmse,lower,0.4624392405112558,-0.4624392405112558,0.4799761534270821,greedy-heads,exp/integrated/sweeps/cooking-time/rla_rank8_noise0.001-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_rank8_noise0.001-greedy-heads-evaluation/0,False,23088 +sweeps,cooking-time,rla_rank8_noise0.0001,0,RMSE,rmse,lower,0.46500539773966587,-0.46500539773966587,0.4826475534492275,best-head,exp/integrated/sweeps/cooking-time/rla_rank8_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_rank8_noise0.0001-best-head-evaluation/0,False,23088 +sweeps,cooking-time,rla_rank8_noise0.0001,0,RMSE,rmse,lower,0.4625085630139665,-0.4625085630139665,0.4800564927891271,greedy-heads,exp/integrated/sweeps/cooking-time/rla_rank8_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_rank8_noise0.0001-greedy-heads-evaluation/0,False,23088 +sweeps,cooking-time,rla_rank8_noise1e-05,0,RMSE,rmse,lower,0.46497304704958375,-0.46497304704958375,0.4824692397862243,best-head,exp/integrated/sweeps/cooking-time/rla_rank8_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_rank8_noise1e-05-best-head-evaluation/0,False,23088 +sweeps,cooking-time,rla_rank8_noise1e-05,0,RMSE,rmse,lower,0.46244718340270563,-0.46244718340270563,0.4799796305248435,greedy-heads,exp/integrated/sweeps/cooking-time/rla_rank8_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_rank8_noise1e-05-greedy-heads-evaluation/0,False,23088 +sweeps,cooking-time,rla_mfb,0,RMSE,rmse,lower,0.46542866555060514,-0.46542866555060514,0.48315113561982953,best-head,exp/integrated/sweeps/cooking-time/rla_mfb-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_mfb-best-head-evaluation/0,False,23088 +sweeps,cooking-time,rla_mfb,0,RMSE,rmse,lower,0.46255113731378883,-0.46255113731378883,0.4799569823688197,greedy-heads,exp/integrated/sweeps/cooking-time/rla_mfb-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_mfb-greedy-heads-evaluation/0,False,23088 +sweeps,cooking-time,esam_cf_fisd,0,RMSE,rmse,lower,0.4639245891190366,-0.4639245891190366,0.48198567488759675,best-head,exp/integrated/sweeps/cooking-time/esam_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/cooking-time/esam_cf_fisd-best-head-evaluation/0,False,17982 +sweeps,cooking-time,esam_cf_fisd,0,RMSE,rmse,lower,0.4629679265241866,-0.4629679265241866,0.4805148498004945,greedy-heads,exp/integrated/sweeps/cooking-time/esam_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/cooking-time/esam_cf_fisd-greedy-heads-evaluation/0,False,17982 +sweeps,cooking-time,esam_mfb,0,RMSE,rmse,lower,0.46541405002003006,-0.46541405002003006,0.4825725860549557,best-head,exp/integrated/sweeps/cooking-time/esam_mfb-evaluation/0.toml,exp/integrated/sweeps/cooking-time/esam_mfb-best-head-evaluation/0,False,23088 +sweeps,cooking-time,esam_mfb,0,RMSE,rmse,lower,0.4625995219609569,-0.4625995219609569,0.47992605869184196,greedy-heads,exp/integrated/sweeps/cooking-time/esam_mfb-evaluation/0.toml,exp/integrated/sweeps/cooking-time/esam_mfb-greedy-heads-evaluation/0,False,23088 +sweeps,cooking-time,all_minus_rla,0,RMSE,rmse,lower,0.4640838759408798,-0.4640838759408798,0.48156976542107044,best-head,exp/integrated/sweeps/cooking-time/all_minus_rla-evaluation/0.toml,exp/integrated/sweeps/cooking-time/all_minus_rla-best-head-evaluation/0,False,17982 +sweeps,cooking-time,all_minus_rla,0,RMSE,rmse,lower,0.46292271885090186,-0.46292271885090186,0.48053390557368486,greedy-heads,exp/integrated/sweeps/cooking-time/all_minus_rla-evaluation/0.toml,exp/integrated/sweeps/cooking-time/all_minus_rla-greedy-heads-evaluation/0,False,17982 +sweeps,cooking-time,rla_cf_fisd,0,RMSE,rmse,lower,0.4638249749501882,-0.4638249749501882,0.4819157837704579,best-head,exp/integrated/sweeps/cooking-time/rla_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_cf_fisd-best-head-evaluation/0,False,17982 +sweeps,cooking-time,rla_cf_fisd,0,RMSE,rmse,lower,0.4627888241622746,-0.4627888241622746,0.480810230139191,greedy-heads,exp/integrated/sweeps/cooking-time/rla_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_cf_fisd-greedy-heads-evaluation/0,False,17982 +sweeps,cooking-time,all_minus_esam,0,RMSE,rmse,lower,0.46397896473985684,-0.46397896473985684,0.481064806640852,best-head,exp/integrated/sweeps/cooking-time/all_minus_esam-evaluation/0.toml,exp/integrated/sweeps/cooking-time/all_minus_esam-best-head-evaluation/0,False,17982 +sweeps,cooking-time,all_minus_esam,0,RMSE,rmse,lower,0.4629167637909319,-0.4629167637909319,0.4803221564231222,greedy-heads,exp/integrated/sweeps/cooking-time/all_minus_esam-evaluation/0.toml,exp/integrated/sweeps/cooking-time/all_minus_esam-greedy-heads-evaluation/0,False,17982 +sweeps,cooking-time,rla_esam,0,RMSE,rmse,lower,0.46492255368357066,-0.46492255368357066,0.4825495500767423,best-head,exp/integrated/sweeps/cooking-time/rla_esam-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_esam-best-head-evaluation/0,False,23088 +sweeps,cooking-time,rla_esam,0,RMSE,rmse,lower,0.4624234187671886,-0.4624234187671886,0.4800586966547807,greedy-heads,exp/integrated/sweeps/cooking-time/rla_esam-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_esam-greedy-heads-evaluation/0,False,23088 +sweeps,cooking-time,all_minus_cf_fisd,0,RMSE,rmse,lower,0.4649621508176366,-0.4649621508176366,0.48187581714110606,best-head,exp/integrated/sweeps/cooking-time/all_minus_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/cooking-time/all_minus_cf_fisd-best-head-evaluation/0,False,17982 +sweeps,cooking-time,all_minus_cf_fisd,0,RMSE,rmse,lower,0.46276401446104365,-0.46276401446104365,0.4800152693247022,greedy-heads,exp/integrated/sweeps/cooking-time/all_minus_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/cooking-time/all_minus_cf_fisd-greedy-heads-evaluation/0,False,17982 +sweeps,cooking-time,all_minus_mfb,0,RMSE,rmse,lower,0.46380663024741176,-0.46380663024741176,0.48139068107848576,best-head,exp/integrated/sweeps/cooking-time/all_minus_mfb-evaluation/0.toml,exp/integrated/sweeps/cooking-time/all_minus_mfb-best-head-evaluation/0,False,17982 +sweeps,cooking-time,all_minus_mfb,0,RMSE,rmse,lower,0.4628484521639483,-0.4628484521639483,0.4805722784138239,greedy-heads,exp/integrated/sweeps/cooking-time/all_minus_mfb-evaluation/0.toml,exp/integrated/sweeps/cooking-time/all_minus_mfb-greedy-heads-evaluation/0,False,17982 +sweeps,cooking-time,all_four_combined,0,RMSE,rmse,lower,0.46391938568359065,-0.46391938568359065,0.48142889277254225,best-head,exp/integrated/sweeps/cooking-time/all_four_combined-evaluation/0.toml,exp/integrated/sweeps/cooking-time/all_four_combined-best-head-evaluation/0,False,17982 +sweeps,cooking-time,all_four_combined,0,RMSE,rmse,lower,0.4629352081332738,-0.4629352081332738,0.4802514336856013,greedy-heads,exp/integrated/sweeps/cooking-time/all_four_combined-evaluation/0.toml,exp/integrated/sweeps/cooking-time/all_four_combined-greedy-heads-evaluation/0,False,17982 +sweeps,delivery-eta,rla_rank2_noise0.001,0,RMSE,rmse,lower,0.5556336245387727,-0.5556336245387727,0.5518591323593222,mean,exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.001-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.001-evaluation/0,False,1365 +sweeps,delivery-eta,rla_rank2_noise0.0001,0,RMSE,rmse,lower,0.5558827645265865,-0.5558827645265865,0.5504866820118107,mean,exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.0001-evaluation/0,False,1365 +sweeps,delivery-eta,rla_rank2_noise1e-05,0,RMSE,rmse,lower,0.5559787764083384,-0.5559787764083384,0.5500968306007196,mean,exp/integrated/sweeps/delivery-eta/rla_rank2_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_rank2_noise1e-05-evaluation/0,False,1365 +sweeps,delivery-eta,rla_rank4_noise0.001,0,RMSE,rmse,lower,0.5564461619356259,-0.5564461619356259,0.5491469465177617,mean,exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.001-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.001-evaluation/0,False,1365 +sweeps,delivery-eta,rla_rank4_noise0.0001,0,RMSE,rmse,lower,0.5556926485294449,-0.5556926485294449,0.5502675684403805,mean,exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.0001-evaluation/0,False,1365 +sweeps,delivery-eta,rla_rank4_noise1e-05,0,RMSE,rmse,lower,0.5560204514474498,-0.5560204514474498,0.5515690033180755,mean,exp/integrated/sweeps/delivery-eta/rla_rank4_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_rank4_noise1e-05-evaluation/0,False,1365 +sweeps,delivery-eta,rla_rank8_noise0.001,0,RMSE,rmse,lower,0.5561609443354546,-0.5561609443354546,0.5482846212731874,mean,exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.001-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.001-evaluation/0,False,1638 +sweeps,delivery-eta,rla_rank8_noise0.0001,0,RMSE,rmse,lower,0.5572424061712593,-0.5572424061712593,0.546917368746242,mean,exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.0001-evaluation/0,False,1365 +sweeps,delivery-eta,rla_rank8_noise1e-05,0,RMSE,rmse,lower,0.5558005167077797,-0.5558005167077797,0.5500678724978622,mean,exp/integrated/sweeps/delivery-eta/rla_rank8_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_rank8_noise1e-05-evaluation/0,False,1365 +sweeps,delivery-eta,esam_rho0.001,0,RMSE,rmse,lower,0.5561053998926777,-0.5561053998926777,0.5493164333767354,mean,exp/integrated/sweeps/delivery-eta/esam_rho0.001-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/esam_rho0.001-evaluation/0,False,1365 +sweeps,delivery-eta,esam_rho0.0025,0,RMSE,rmse,lower,0.5559746489354217,-0.5559746489354217,0.5535077457230605,mean,exp/integrated/sweeps/delivery-eta/esam_rho0.0025-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/esam_rho0.0025-evaluation/0,False,2184 +sweeps,delivery-eta,esam_rho0.005,0,RMSE,rmse,lower,0.5556360918188752,-0.5556360918188752,0.5494649592968895,mean,exp/integrated/sweeps/delivery-eta/esam_rho0.005-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/esam_rho0.005-evaluation/0,False,2184 +sweeps,delivery-eta,esam_rho0.01,0,RMSE,rmse,lower,0.555695893192201,-0.555695893192201,0.5503082950221266,mean,exp/integrated/sweeps/delivery-eta/esam_rho0.01-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/esam_rho0.01-evaluation/0,False,1365 +sweeps,delivery-eta,mfb_keep0.7,0,RMSE,rmse,lower,0.5576335149669441,-0.5576335149669441,0.5507854163911843,mean,exp/integrated/sweeps/delivery-eta/mfb_keep0.7-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/mfb_keep0.7-evaluation/0,False,2730 +sweeps,delivery-eta,mfb_keep0.8,0,RMSE,rmse,lower,0.5567305910455574,-0.5567305910455574,0.5489749915571137,mean,exp/integrated/sweeps/delivery-eta/mfb_keep0.8-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/mfb_keep0.8-evaluation/0,False,2730 +sweeps,delivery-eta,mfb_keep0.9,0,RMSE,rmse,lower,0.5560026829864069,-0.5560026829864069,0.5501201530906976,mean,exp/integrated/sweeps/delivery-eta/mfb_keep0.9-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/mfb_keep0.9-evaluation/0,False,2184 +sweeps,delivery-eta,mfb_keep0.95,0,RMSE,rmse,lower,0.5557158166151223,-0.5557158166151223,0.5496765041498511,mean,exp/integrated/sweeps/delivery-eta/mfb_keep0.95-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/mfb_keep0.95-evaluation/0,False,2184 +sweeps,delivery-eta,cf_fisd_lambda0.05,0,RMSE,rmse,lower,0.5562445854609275,-0.5562445854609275,0.5504166498793033,mean,exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.05-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.05-evaluation/0,False,2184 +sweeps,delivery-eta,cf_fisd_lambda0.1,0,RMSE,rmse,lower,0.5562691235201772,-0.5562691235201772,0.5515996385557942,mean,exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.1-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.1-evaluation/0,False,3003 +sweeps,delivery-eta,cf_fisd_lambda0.2,0,RMSE,rmse,lower,0.5565664676993906,-0.5565664676993906,0.5475279860550825,mean,exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.2-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.2-evaluation/0,False,1638 +sweeps,delivery-eta,rla_esam,0,RMSE,rmse,lower,0.5559417888696784,-0.5559417888696784,0.5541331785404355,mean,exp/integrated/sweeps/delivery-eta/rla_esam-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_esam-evaluation/0,False,1365 +sweeps,delivery-eta,rla_mfb,0,RMSE,rmse,lower,0.5559017430643394,-0.5559017430643394,0.5492186599232151,mean,exp/integrated/sweeps/delivery-eta/rla_mfb-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_mfb-evaluation/0,False,1365 +sweeps,delivery-eta,rla_cf_fisd,0,RMSE,rmse,lower,0.556067616855386,-0.556067616855386,0.5512732344334902,mean,exp/integrated/sweeps/delivery-eta/rla_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_cf_fisd-evaluation/0,False,1638 +sweeps,delivery-eta,esam_mfb,0,RMSE,rmse,lower,0.5560703233852616,-0.5560703233852616,0.5508616503748457,mean,exp/integrated/sweeps/delivery-eta/esam_mfb-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/esam_mfb-evaluation/0,False,2730 +sweeps,delivery-eta,esam_cf_fisd,0,RMSE,rmse,lower,0.5562165100217151,-0.5562165100217151,0.5500920359593422,mean,exp/integrated/sweeps/delivery-eta/esam_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/esam_cf_fisd-evaluation/0,False,1638 +sweeps,delivery-eta,mfb_cf_fisd,0,RMSE,rmse,lower,0.55593889408891,-0.55593889408891,0.5537058512136184,mean,exp/integrated/sweeps/delivery-eta/mfb_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/mfb_cf_fisd-evaluation/0,False,1638 +sweeps,delivery-eta,all_four_combined,0,RMSE,rmse,lower,0.5560789252392828,-0.5560789252392828,0.5493995976501785,mean,exp/integrated/sweeps/delivery-eta/all_four_combined-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/all_four_combined-evaluation/0,False,1638 +sweeps,delivery-eta,all_minus_rla,0,RMSE,rmse,lower,0.5560596579733861,-0.5560596579733861,0.5513667248174312,mean,exp/integrated/sweeps/delivery-eta/all_minus_rla-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/all_minus_rla-evaluation/0,False,1638 +sweeps,delivery-eta,all_minus_esam,0,RMSE,rmse,lower,0.556102345189303,-0.556102345189303,0.5505710768952337,mean,exp/integrated/sweeps/delivery-eta/all_minus_esam-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/all_minus_esam-evaluation/0,False,1365 +sweeps,delivery-eta,all_minus_mfb,0,RMSE,rmse,lower,0.5571737580629273,-0.5571737580629273,0.5527981832493051,mean,exp/integrated/sweeps/delivery-eta/all_minus_mfb-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/all_minus_mfb-evaluation/0,False,2184 +sweeps,delivery-eta,all_minus_cf_fisd,0,RMSE,rmse,lower,0.5563682291205534,-0.5563682291205534,0.5506428213190716,mean,exp/integrated/sweeps/delivery-eta/all_minus_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/all_minus_cf_fisd-evaluation/0,False,2184 +sweeps,delivery-eta,rla_rank2_noise0.001,0,RMSE,rmse,lower,0.5574634025946379,-0.5574634025946379,0.5547066537478277,best-head,exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.001-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.001-best-head-evaluation/0,False,1365 +sweeps,delivery-eta,rla_rank2_noise0.001,0,RMSE,rmse,lower,0.5552720495549479,-0.5552720495549479,0.5524877260752137,greedy-heads,exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.001-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.001-greedy-heads-evaluation/0,False,1365 +sweeps,delivery-eta,rla_rank2_noise0.0001,0,RMSE,rmse,lower,0.5574308976082757,-0.5574308976082757,0.5550464489547554,best-head,exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.0001-best-head-evaluation/0,False,1365 +sweeps,delivery-eta,rla_rank2_noise0.0001,0,RMSE,rmse,lower,0.5553755991930566,-0.5553755991930566,0.5516112816611277,greedy-heads,exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.0001-greedy-heads-evaluation/0,False,1365 +sweeps,delivery-eta,rla_rank2_noise1e-05,0,RMSE,rmse,lower,0.5574473374558679,-0.5574473374558679,0.5570882235759158,best-head,exp/integrated/sweeps/delivery-eta/rla_rank2_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_rank2_noise1e-05-best-head-evaluation/0,False,1365 +sweeps,delivery-eta,rla_rank2_noise1e-05,0,RMSE,rmse,lower,0.5553479091365543,-0.5553479091365543,0.5511482044438852,greedy-heads,exp/integrated/sweeps/delivery-eta/rla_rank2_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_rank2_noise1e-05-greedy-heads-evaluation/0,False,1365 +sweeps,delivery-eta,rla_rank4_noise0.001,0,RMSE,rmse,lower,0.5571872637155045,-0.5571872637155045,0.5515561165588831,best-head,exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.001-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.001-best-head-evaluation/0,False,1365 +sweeps,delivery-eta,rla_rank4_noise0.001,0,RMSE,rmse,lower,0.5552950473462183,-0.5552950473462183,0.5512393912767233,greedy-heads,exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.001-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.001-greedy-heads-evaluation/0,False,1365 +sweeps,delivery-eta,rla_rank4_noise0.0001,0,RMSE,rmse,lower,0.5573103773832353,-0.5573103773832353,0.5532468973961396,best-head,exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.0001-best-head-evaluation/0,False,1365 +sweeps,delivery-eta,rla_rank4_noise0.0001,0,RMSE,rmse,lower,0.5553051102575359,-0.5553051102575359,0.5508335169889497,greedy-heads,exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.0001-greedy-heads-evaluation/0,False,1365 +sweeps,delivery-eta,esam_rho0.001,0,RMSE,rmse,lower,0.5577754983463842,-0.5577754983463842,0.5539871683993082,best-head,exp/integrated/sweeps/delivery-eta/esam_rho0.001-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/esam_rho0.001-best-head-evaluation/0,False,1365 +sweeps,delivery-eta,esam_rho0.001,0,RMSE,rmse,lower,0.5555279503030305,-0.5555279503030305,0.5511287648017675,greedy-heads,exp/integrated/sweeps/delivery-eta/esam_rho0.001-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/esam_rho0.001-greedy-heads-evaluation/0,False,1365 +sweeps,delivery-eta,rla_rank4_noise1e-05,0,RMSE,rmse,lower,0.5569447272303437,-0.5569447272303437,0.5529624818746817,best-head,exp/integrated/sweeps/delivery-eta/rla_rank4_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_rank4_noise1e-05-best-head-evaluation/0,False,1365 +sweeps,delivery-eta,rla_rank4_noise1e-05,0,RMSE,rmse,lower,0.5553112552523684,-0.5553112552523684,0.552273210826067,greedy-heads,exp/integrated/sweeps/delivery-eta/rla_rank4_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_rank4_noise1e-05-greedy-heads-evaluation/0,False,1365 +sweeps,delivery-eta,esam_rho0.0025,0,RMSE,rmse,lower,0.5594097126680496,-0.5594097126680496,0.5568830263477744,best-head,exp/integrated/sweeps/delivery-eta/esam_rho0.0025-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/esam_rho0.0025-best-head-evaluation/0,False,2184 +sweeps,delivery-eta,esam_rho0.0025,0,RMSE,rmse,lower,0.5555843034564808,-0.5555843034564808,0.5528859176747987,greedy-heads,exp/integrated/sweeps/delivery-eta/esam_rho0.0025-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/esam_rho0.0025-greedy-heads-evaluation/0,False,2184 +sweeps,delivery-eta,esam_rho0.005,0,RMSE,rmse,lower,0.5582863288518123,-0.5582863288518123,0.5560390501031047,best-head,exp/integrated/sweeps/delivery-eta/esam_rho0.005-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/esam_rho0.005-best-head-evaluation/0,False,2184 +sweeps,delivery-eta,esam_rho0.005,0,RMSE,rmse,lower,0.5551513290724005,-0.5551513290724005,0.5507488105906536,greedy-heads,exp/integrated/sweeps/delivery-eta/esam_rho0.005-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/esam_rho0.005-greedy-heads-evaluation/0,False,2184 +sweeps,delivery-eta,esam_rho0.01,0,RMSE,rmse,lower,0.5577866919608585,-0.5577866919608585,0.5538906498744642,best-head,exp/integrated/sweeps/delivery-eta/esam_rho0.01-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/esam_rho0.01-best-head-evaluation/0,False,1365 +sweeps,delivery-eta,esam_rho0.01,0,RMSE,rmse,lower,0.5553495995574796,-0.5553495995574796,0.5513616169057922,greedy-heads,exp/integrated/sweeps/delivery-eta/esam_rho0.01-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/esam_rho0.01-greedy-heads-evaluation/0,False,1365 +sweeps,delivery-eta,rla_rank8_noise0.0001,0,RMSE,rmse,lower,0.5573390661266857,-0.5573390661266857,0.5535791095517189,best-head,exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.0001-best-head-evaluation/0,False,1365 +sweeps,delivery-eta,rla_rank8_noise0.0001,0,RMSE,rmse,lower,0.5558846409645507,-0.5558846409645507,0.5506559188769905,greedy-heads,exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.0001-greedy-heads-evaluation/0,False,1365 +sweeps,delivery-eta,rla_rank8_noise0.001,0,RMSE,rmse,lower,0.5576981787979653,-0.5576981787979653,0.5514806270620619,best-head,exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.001-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.001-best-head-evaluation/0,False,1638 +sweeps,delivery-eta,rla_rank8_noise0.001,0,RMSE,rmse,lower,0.5556463362777673,-0.5556463362777673,0.5491140035096563,greedy-heads,exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.001-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.001-greedy-heads-evaluation/0,False,1638 +sweeps,delivery-eta,rla_rank8_noise1e-05,0,RMSE,rmse,lower,0.5574517747950514,-0.5574517747950514,0.5534907850292722,best-head,exp/integrated/sweeps/delivery-eta/rla_rank8_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_rank8_noise1e-05-best-head-evaluation/0,False,1365 +sweeps,delivery-eta,rla_rank8_noise1e-05,0,RMSE,rmse,lower,0.5553793554897237,-0.5553793554897237,0.5521969288494308,greedy-heads,exp/integrated/sweeps/delivery-eta/rla_rank8_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_rank8_noise1e-05-greedy-heads-evaluation/0,False,1365 +sweeps,delivery-eta,cf_fisd_lambda0.2,0,RMSE,rmse,lower,0.5568247172798395,-0.5568247172798395,0.5526535729970853,best-head,exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.2-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.2-best-head-evaluation/0,False,1638 +sweeps,delivery-eta,cf_fisd_lambda0.2,0,RMSE,rmse,lower,0.5558508909198594,-0.5558508909198594,0.5511616144232316,greedy-heads,exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.2-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.2-greedy-heads-evaluation/0,False,1638 +sweeps,delivery-eta,cf_fisd_lambda0.05,0,RMSE,rmse,lower,0.557661572508718,-0.557661572508718,0.553144673358601,best-head,exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.05-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.05-best-head-evaluation/0,False,2184 +sweeps,delivery-eta,cf_fisd_lambda0.05,0,RMSE,rmse,lower,0.5556691040453957,-0.5556691040453957,0.5521822756750591,greedy-heads,exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.05-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.05-greedy-heads-evaluation/0,False,2184 +sweeps,delivery-eta,mfb_cf_fisd,0,RMSE,rmse,lower,0.556619663847731,-0.556619663847731,0.55593546321811,best-head,exp/integrated/sweeps/delivery-eta/mfb_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/mfb_cf_fisd-best-head-evaluation/0,False,1638 +sweeps,delivery-eta,mfb_cf_fisd,0,RMSE,rmse,lower,0.5555428639260286,-0.5555428639260286,0.5513865344186518,greedy-heads,exp/integrated/sweeps/delivery-eta/mfb_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/mfb_cf_fisd-greedy-heads-evaluation/0,False,1638 +sweeps,delivery-eta,mfb_keep0.95,0,RMSE,rmse,lower,0.5584568843028204,-0.5584568843028204,0.5549022634897627,best-head,exp/integrated/sweeps/delivery-eta/mfb_keep0.95-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/mfb_keep0.95-best-head-evaluation/0,False,2184 +sweeps,delivery-eta,mfb_keep0.9,0,RMSE,rmse,lower,0.5590040394051904,-0.5590040394051904,0.5557457406296488,best-head,exp/integrated/sweeps/delivery-eta/mfb_keep0.9-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/mfb_keep0.9-best-head-evaluation/0,False,2184 +sweeps,delivery-eta,mfb_keep0.95,0,RMSE,rmse,lower,0.5552807442811304,-0.5552807442811304,0.5512821544083845,greedy-heads,exp/integrated/sweeps/delivery-eta/mfb_keep0.95-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/mfb_keep0.95-greedy-heads-evaluation/0,False,2184 +sweeps,delivery-eta,mfb_keep0.9,0,RMSE,rmse,lower,0.5555016895512279,-0.5555016895512279,0.5503428181880856,greedy-heads,exp/integrated/sweeps/delivery-eta/mfb_keep0.9-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/mfb_keep0.9-greedy-heads-evaluation/0,False,2184 +sweeps,delivery-eta,cf_fisd_lambda0.1,0,RMSE,rmse,lower,0.5580564462352832,-0.5580564462352832,0.5528824139554394,best-head,exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.1-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.1-best-head-evaluation/0,False,3003 +sweeps,delivery-eta,cf_fisd_lambda0.1,0,RMSE,rmse,lower,0.5559326756199947,-0.5559326756199947,0.5522105562224877,greedy-heads,exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.1-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.1-greedy-heads-evaluation/0,False,3003 +sweeps,delivery-eta,mfb_keep0.7,0,RMSE,rmse,lower,0.5595885535124743,-0.5595885535124743,0.5534224254724212,best-head,exp/integrated/sweeps/delivery-eta/mfb_keep0.7-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/mfb_keep0.7-best-head-evaluation/0,False,2730 +sweeps,delivery-eta,mfb_keep0.7,0,RMSE,rmse,lower,0.5551008108444078,-0.5551008108444078,0.5501919833980724,greedy-heads,exp/integrated/sweeps/delivery-eta/mfb_keep0.7-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/mfb_keep0.7-greedy-heads-evaluation/0,False,2730 +sweeps,delivery-eta,mfb_keep0.8,0,RMSE,rmse,lower,0.5594878077640214,-0.5594878077640214,0.5558040556519808,best-head,exp/integrated/sweeps/delivery-eta/mfb_keep0.8-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/mfb_keep0.8-best-head-evaluation/0,False,2730 +sweeps,delivery-eta,mfb_keep0.8,0,RMSE,rmse,lower,0.5554990875501031,-0.5554990875501031,0.5493161621093208,greedy-heads,exp/integrated/sweeps/delivery-eta/mfb_keep0.8-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/mfb_keep0.8-greedy-heads-evaluation/0,False,2730 +sweeps,delivery-eta,esam_cf_fisd,0,RMSE,rmse,lower,0.5568538056781229,-0.5568538056781229,0.5520497317854985,best-head,exp/integrated/sweeps/delivery-eta/esam_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/esam_cf_fisd-best-head-evaluation/0,False,1638 +sweeps,delivery-eta,esam_cf_fisd,0,RMSE,rmse,lower,0.5558497381815869,-0.5558497381815869,0.5508600814353861,greedy-heads,exp/integrated/sweeps/delivery-eta/esam_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/esam_cf_fisd-greedy-heads-evaluation/0,False,1638 +sweeps,delivery-eta,all_minus_rla,0,RMSE,rmse,lower,0.5572234465535005,-0.5572234465535005,0.5525491087220346,best-head,exp/integrated/sweeps/delivery-eta/all_minus_rla-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/all_minus_rla-best-head-evaluation/0,False,1638 +sweeps,delivery-eta,all_minus_rla,0,RMSE,rmse,lower,0.5556365745463094,-0.5556365745463094,0.5515955863733182,greedy-heads,exp/integrated/sweeps/delivery-eta/all_minus_rla-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/all_minus_rla-greedy-heads-evaluation/0,False,1638 +sweeps,delivery-eta,esam_mfb,0,RMSE,rmse,lower,0.5602888251013873,-0.5602888251013873,0.5556835311938418,best-head,exp/integrated/sweeps/delivery-eta/esam_mfb-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/esam_mfb-best-head-evaluation/0,False,2730 +sweeps,delivery-eta,esam_mfb,0,RMSE,rmse,lower,0.5556538720026133,-0.5556538720026133,0.5520326723090689,greedy-heads,exp/integrated/sweeps/delivery-eta/esam_mfb-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/esam_mfb-greedy-heads-evaluation/0,False,2730 +sweeps,delivery-eta,rla_cf_fisd,0,RMSE,rmse,lower,0.5567133805762534,-0.5567133805762534,0.5518657747493417,best-head,exp/integrated/sweeps/delivery-eta/rla_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_cf_fisd-best-head-evaluation/0,False,1638 +sweeps,delivery-eta,rla_cf_fisd,0,RMSE,rmse,lower,0.55586820851409,-0.55586820851409,0.5514142072129953,greedy-heads,exp/integrated/sweeps/delivery-eta/rla_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_cf_fisd-greedy-heads-evaluation/0,False,1638 +sweeps,delivery-eta,rla_mfb,0,RMSE,rmse,lower,0.5579516581852805,-0.5579516581852805,0.5533480523154869,best-head,exp/integrated/sweeps/delivery-eta/rla_mfb-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_mfb-best-head-evaluation/0,False,1365 +sweeps,delivery-eta,rla_mfb,0,RMSE,rmse,lower,0.5554524372350571,-0.5554524372350571,0.5505269863779684,greedy-heads,exp/integrated/sweeps/delivery-eta/rla_mfb-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_mfb-greedy-heads-evaluation/0,False,1365 +sweeps,delivery-eta,all_minus_esam,0,RMSE,rmse,lower,0.5574872455078781,-0.5574872455078781,0.5557374285650164,best-head,exp/integrated/sweeps/delivery-eta/all_minus_esam-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/all_minus_esam-best-head-evaluation/0,False,1365 +sweeps,delivery-eta,all_minus_esam,0,RMSE,rmse,lower,0.5556806886998003,-0.5556806886998003,0.5513624006625033,greedy-heads,exp/integrated/sweeps/delivery-eta/all_minus_esam-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/all_minus_esam-greedy-heads-evaluation/0,False,1365 +sweeps,delivery-eta,rla_esam,0,RMSE,rmse,lower,0.5572907783524989,-0.5572907783524989,0.5545303498210034,best-head,exp/integrated/sweeps/delivery-eta/rla_esam-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_esam-best-head-evaluation/0,False,1365 +sweeps,delivery-eta,rla_esam,0,RMSE,rmse,lower,0.5552988041877751,-0.5552988041877751,0.5525187688548042,greedy-heads,exp/integrated/sweeps/delivery-eta/rla_esam-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_esam-greedy-heads-evaluation/0,False,1365 +sweeps,delivery-eta,all_four_combined,0,RMSE,rmse,lower,0.557107642501001,-0.557107642501001,0.5520572896128711,best-head,exp/integrated/sweeps/delivery-eta/all_four_combined-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/all_four_combined-best-head-evaluation/0,False,1638 +sweeps,delivery-eta,all_four_combined,0,RMSE,rmse,lower,0.5558210261849236,-0.5558210261849236,0.551614874495521,greedy-heads,exp/integrated/sweeps/delivery-eta/all_four_combined-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/all_four_combined-greedy-heads-evaluation/0,False,1638 +sweeps,delivery-eta,all_minus_mfb,0,RMSE,rmse,lower,0.5591485795972553,-0.5591485795972553,0.5570234889363207,best-head,exp/integrated/sweeps/delivery-eta/all_minus_mfb-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/all_minus_mfb-best-head-evaluation/0,False,2184 +sweeps,delivery-eta,all_minus_mfb,0,RMSE,rmse,lower,0.5566172544693042,-0.5566172544693042,0.5524610510858833,greedy-heads,exp/integrated/sweeps/delivery-eta/all_minus_mfb-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/all_minus_mfb-greedy-heads-evaluation/0,False,2184 +sweeps,delivery-eta,all_minus_cf_fisd,0,RMSE,rmse,lower,0.5589936165675472,-0.5589936165675472,0.5554125439499674,best-head,exp/integrated/sweeps/delivery-eta/all_minus_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/all_minus_cf_fisd-best-head-evaluation/0,False,2184 +sweeps,delivery-eta,all_minus_cf_fisd,0,RMSE,rmse,lower,0.5557239681192615,-0.5557239681192615,0.5525121073453494,greedy-heads,exp/integrated/sweeps/delivery-eta/all_minus_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/all_minus_cf_fisd-greedy-heads-evaluation/0,False,2184 +final,sberbank-housing,baseline_plr,0,RMSE,rmse,lower,0.23452539387100468,-0.23452539387100468,0.23846659721529556,mean,exp/integrated/final/sberbank-housing/baseline_plr-evaluation/0.toml,exp/integrated/final/sberbank-housing/baseline_plr-evaluation/0,False,814 +final,sberbank-housing,baseline_plr,2,RMSE,rmse,lower,0.23581900020763122,-0.23581900020763122,0.23293721743587778,mean,exp/integrated/final/sberbank-housing/baseline_plr-evaluation/2.toml,exp/integrated/final/sberbank-housing/baseline_plr-evaluation/2,False,666 +final,sberbank-housing,baseline_plr,1,RMSE,rmse,lower,0.23623006536000674,-0.23623006536000674,0.2318497676950212,mean,exp/integrated/final/sberbank-housing/baseline_plr-evaluation/1.toml,exp/integrated/final/sberbank-housing/baseline_plr-evaluation/1,False,962 +final,sberbank-housing,best_rla_only,0,RMSE,rmse,lower,0.2354682854519358,-0.2354682854519358,0.24014251007009368,mean,exp/integrated/final/sberbank-housing/best_rla_only-evaluation/0.toml,exp/integrated/final/sberbank-housing/best_rla_only-evaluation/0,False,814 +final,sberbank-housing,best_rla_only,1,RMSE,rmse,lower,0.23689852205208367,-0.23689852205208367,0.23132638249744272,mean,exp/integrated/final/sberbank-housing/best_rla_only-evaluation/1.toml,exp/integrated/final/sberbank-housing/best_rla_only-evaluation/1,False,814 +final,sberbank-housing,best_rla_only,2,RMSE,rmse,lower,0.2359145620703149,-0.2359145620703149,0.2332252993496603,mean,exp/integrated/final/sberbank-housing/best_rla_only-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_rla_only-evaluation/2,False,666 +final,sberbank-housing,best_esam_only,0,RMSE,rmse,lower,0.23474623366768377,-0.23474623366768377,0.23936563514587833,mean,exp/integrated/final/sberbank-housing/best_esam_only-evaluation/0.toml,exp/integrated/final/sberbank-housing/best_esam_only-evaluation/0,False,814 +final,sberbank-housing,best_esam_only,2,RMSE,rmse,lower,0.2356560348188032,-0.2356560348188032,0.23118945759761478,mean,exp/integrated/final/sberbank-housing/best_esam_only-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_esam_only-evaluation/2,False,814 +final,sberbank-housing,best_esam_only,1,RMSE,rmse,lower,0.23723213436744667,-0.23723213436744667,0.23099161094856513,mean,exp/integrated/final/sberbank-housing/best_esam_only-evaluation/1.toml,exp/integrated/final/sberbank-housing/best_esam_only-evaluation/1,False,814 +final,sberbank-housing,best_mfb_only,0,RMSE,rmse,lower,0.2341279953511937,-0.2341279953511937,0.23402994505001048,mean,exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/0.toml,exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/0,False,888 +final,sberbank-housing,best_mfb_only,1,RMSE,rmse,lower,0.23621923128975542,-0.23621923128975542,0.23276193568912745,mean,exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/1.toml,exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/1,False,592 +final,sberbank-housing,best_mfb_only,2,RMSE,rmse,lower,0.23605616345434166,-0.23605616345434166,0.23451195530114213,mean,exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/2,False,1110 +final,sberbank-housing,best_cf_fisd_only,0,RMSE,rmse,lower,0.23523413233394516,-0.23523413233394516,0.235149605552611,mean,exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/0.toml,exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/0,False,666 +final,sberbank-housing,best_cf_fisd_only,1,RMSE,rmse,lower,0.23639255622691643,-0.23639255622691643,0.2324256095265638,mean,exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/1.toml,exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/1,False,814 +final,sberbank-housing,best_cf_fisd_only,2,RMSE,rmse,lower,0.2359054032023901,-0.2359054032023901,0.23245970634502372,mean,exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/2,False,518 +final,sberbank-housing,best_combined,0,RMSE,rmse,lower,0.235050071622046,-0.235050071622046,0.23184680318718037,mean,exp/integrated/final/sberbank-housing/best_combined-evaluation/0.toml,exp/integrated/final/sberbank-housing/best_combined-evaluation/0,False,666 +final,sberbank-housing,best_combined,1,RMSE,rmse,lower,0.23618618194018487,-0.23618618194018487,0.23253236276211406,mean,exp/integrated/final/sberbank-housing/best_combined-evaluation/1.toml,exp/integrated/final/sberbank-housing/best_combined-evaluation/1,False,1184 +final,sberbank-housing,best_combined,2,RMSE,rmse,lower,0.23592642069691294,-0.23592642069691294,0.2326191697888986,mean,exp/integrated/final/sberbank-housing/best_combined-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_combined-evaluation/2,False,888 +final,sberbank-housing,baseline_plr,0,RMSE,rmse,lower,0.23955454428788905,-0.23955454428788905,0.26588264760298186,best-head,exp/integrated/final/sberbank-housing/baseline_plr-evaluation/0.toml,exp/integrated/final/sberbank-housing/baseline_plr-best-head-evaluation/0,False,814 +final,sberbank-housing,baseline_plr,2,RMSE,rmse,lower,0.23944956044068919,-0.23944956044068919,0.24308412455882378,best-head,exp/integrated/final/sberbank-housing/baseline_plr-evaluation/2.toml,exp/integrated/final/sberbank-housing/baseline_plr-best-head-evaluation/2,False,666 +final,sberbank-housing,baseline_plr,1,RMSE,rmse,lower,0.24150787640249508,-0.24150787640249508,0.2416341058127328,best-head,exp/integrated/final/sberbank-housing/baseline_plr-evaluation/1.toml,exp/integrated/final/sberbank-housing/baseline_plr-best-head-evaluation/1,False,962 +final,sberbank-housing,baseline_plr,0,RMSE,rmse,lower,0.23257941028161666,-0.23257941028161666,0.23549883338311248,greedy-heads,exp/integrated/final/sberbank-housing/baseline_plr-evaluation/0.toml,exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/0,False,814 +final,sberbank-housing,baseline_plr,2,RMSE,rmse,lower,0.23449523543792233,-0.23449523543792233,0.23591672540617004,greedy-heads,exp/integrated/final/sberbank-housing/baseline_plr-evaluation/2.toml,exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/2,False,666 +final,sberbank-housing,baseline_plr,1,RMSE,rmse,lower,0.2341956008327552,-0.2341956008327552,0.23089961816224172,greedy-heads,exp/integrated/final/sberbank-housing/baseline_plr-evaluation/1.toml,exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/1,False,962 +final,sberbank-housing,best_rla_only,0,RMSE,rmse,lower,0.2370483756746748,-0.2370483756746748,0.23920040609688478,best-head,exp/integrated/final/sberbank-housing/best_rla_only-evaluation/0.toml,exp/integrated/final/sberbank-housing/best_rla_only-best-head-evaluation/0,False,814 +final,sberbank-housing,best_rla_only,2,RMSE,rmse,lower,0.2381416834471136,-0.2381416834471136,0.2498549025398126,best-head,exp/integrated/final/sberbank-housing/best_rla_only-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_rla_only-best-head-evaluation/2,False,666 +final,sberbank-housing,best_rla_only,1,RMSE,rmse,lower,0.23898563084143007,-0.23898563084143007,0.23816985512587505,best-head,exp/integrated/final/sberbank-housing/best_rla_only-evaluation/1.toml,exp/integrated/final/sberbank-housing/best_rla_only-best-head-evaluation/1,False,814 +final,sberbank-housing,best_rla_only,0,RMSE,rmse,lower,0.2328613038082034,-0.2328613038082034,0.23498425768312764,greedy-heads,exp/integrated/final/sberbank-housing/best_rla_only-evaluation/0.toml,exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/0,False,814 +final,sberbank-housing,best_rla_only,2,RMSE,rmse,lower,0.23410085698681354,-0.23410085698681354,0.23135944978887066,greedy-heads,exp/integrated/final/sberbank-housing/best_rla_only-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/2,False,666 +final,sberbank-housing,best_rla_only,1,RMSE,rmse,lower,0.23460022912632159,-0.23460022912632159,0.23076016380764466,greedy-heads,exp/integrated/final/sberbank-housing/best_rla_only-evaluation/1.toml,exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/1,False,814 +final,sberbank-housing,best_esam_only,0,RMSE,rmse,lower,0.2377136850270048,-0.2377136850270048,0.262345673404661,best-head,exp/integrated/final/sberbank-housing/best_esam_only-evaluation/0.toml,exp/integrated/final/sberbank-housing/best_esam_only-best-head-evaluation/0,False,814 +final,sberbank-housing,best_esam_only,2,RMSE,rmse,lower,0.24089231567126032,-0.24089231567126032,0.24860647860679874,best-head,exp/integrated/final/sberbank-housing/best_esam_only-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_esam_only-best-head-evaluation/2,False,814 +final,sberbank-housing,best_esam_only,1,RMSE,rmse,lower,0.2385787982165438,-0.2385787982165438,0.24822352719647534,best-head,exp/integrated/final/sberbank-housing/best_esam_only-evaluation/1.toml,exp/integrated/final/sberbank-housing/best_esam_only-best-head-evaluation/1,False,814 +final,sberbank-housing,best_esam_only,0,RMSE,rmse,lower,0.23295802298295287,-0.23295802298295287,0.23903366020873654,greedy-heads,exp/integrated/final/sberbank-housing/best_esam_only-evaluation/0.toml,exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/0,False,814 +final,sberbank-housing,best_esam_only,2,RMSE,rmse,lower,0.23456056719125185,-0.23456056719125185,0.23191054380220286,greedy-heads,exp/integrated/final/sberbank-housing/best_esam_only-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/2,False,814 +final,sberbank-housing,best_esam_only,1,RMSE,rmse,lower,0.23392588182013857,-0.23392588182013857,0.23227402448302542,greedy-heads,exp/integrated/final/sberbank-housing/best_esam_only-evaluation/1.toml,exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/1,False,814 +final,sberbank-housing,best_mfb_only,0,RMSE,rmse,lower,0.2380346366558266,-0.2380346366558266,0.2579307429494123,best-head,exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/0.toml,exp/integrated/final/sberbank-housing/best_mfb_only-best-head-evaluation/0,False,888 +final,sberbank-housing,best_mfb_only,1,RMSE,rmse,lower,0.2385396806698102,-0.2385396806698102,0.2407010051283554,best-head,exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/1.toml,exp/integrated/final/sberbank-housing/best_mfb_only-best-head-evaluation/1,False,592 +final,sberbank-housing,best_mfb_only,2,RMSE,rmse,lower,0.24181054374010003,-0.24181054374010003,0.24863506025450496,best-head,exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_mfb_only-best-head-evaluation/2,False,1110 +final,sberbank-housing,best_mfb_only,0,RMSE,rmse,lower,0.23237596587359466,-0.23237596587359466,0.23064696162155762,greedy-heads,exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/0.toml,exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/0,False,888 +final,sberbank-housing,best_mfb_only,1,RMSE,rmse,lower,0.23377133352416926,-0.23377133352416926,0.23643993065905872,greedy-heads,exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/1.toml,exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/1,False,592 +final,sberbank-housing,best_mfb_only,2,RMSE,rmse,lower,0.23350988305065445,-0.23350988305065445,0.23821200474658916,greedy-heads,exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/2,False,1110 +final,sberbank-housing,best_cf_fisd_only,0,RMSE,rmse,lower,0.2388197411165918,-0.2388197411165918,0.2462523116085554,best-head,exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/0.toml,exp/integrated/final/sberbank-housing/best_cf_fisd_only-best-head-evaluation/0,False,666 +final,sberbank-housing,best_cf_fisd_only,2,RMSE,rmse,lower,0.2376592052308581,-0.2376592052308581,0.23311093711112285,best-head,exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_cf_fisd_only-best-head-evaluation/2,False,518 +final,sberbank-housing,best_cf_fisd_only,1,RMSE,rmse,lower,0.24061445857155628,-0.24061445857155628,0.23970045288478414,best-head,exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/1.toml,exp/integrated/final/sberbank-housing/best_cf_fisd_only-best-head-evaluation/1,False,814 +final,sberbank-housing,best_cf_fisd_only,0,RMSE,rmse,lower,0.23327114512588593,-0.23327114512588593,0.23367141178994547,greedy-heads,exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/0.toml,exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/0,False,666 +final,sberbank-housing,best_cf_fisd_only,2,RMSE,rmse,lower,0.23432768503071347,-0.23432768503071347,0.2325014731401514,greedy-heads,exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/2,False,518 +final,sberbank-housing,best_cf_fisd_only,1,RMSE,rmse,lower,0.23463880474397839,-0.23463880474397839,0.2331063026442073,greedy-heads,exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/1.toml,exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/1,False,814 +final,sberbank-housing,best_combined,0,RMSE,rmse,lower,0.23560357747729638,-0.23560357747729638,0.2389191702722735,best-head,exp/integrated/final/sberbank-housing/best_combined-evaluation/0.toml,exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/0,False,666 +final,sberbank-housing,best_combined,1,RMSE,rmse,lower,0.24254141605050564,-0.24254141605050564,0.24204913155172003,best-head,exp/integrated/final/sberbank-housing/best_combined-evaluation/1.toml,exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/1,False,1184 +final,sberbank-housing,best_combined,2,RMSE,rmse,lower,0.23847619665119374,-0.23847619665119374,0.2542706881867437,best-head,exp/integrated/final/sberbank-housing/best_combined-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/2,False,888 +final,sberbank-housing,best_combined,0,RMSE,rmse,lower,0.2324412121462238,-0.2324412121462238,0.23820740696857753,greedy-heads,exp/integrated/final/sberbank-housing/best_combined-evaluation/0.toml,exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/0,False,666 +final,sberbank-housing,best_combined,1,RMSE,rmse,lower,0.23397077829789442,-0.23397077829789442,0.23276530465893142,greedy-heads,exp/integrated/final/sberbank-housing/best_combined-evaluation/1.toml,exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/1,False,1184 +final,sberbank-housing,best_combined,2,RMSE,rmse,lower,0.23411027740292373,-0.23411027740292373,0.24496736545125247,greedy-heads,exp/integrated/final/sberbank-housing/best_combined-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/2,False,888 +final,ecom-offers,baseline_plr,0,AUROC,roc-auc,higher,0.6227054517511603,0.6227054517511603,0.591151981311678,mean,exp/integrated/final/ecom-offers/baseline_plr-evaluation/0.toml,exp/integrated/final/ecom-offers/baseline_plr-evaluation/0,False,321 +final,ecom-offers,baseline_plr,2,AUROC,roc-auc,higher,0.6212503263524413,0.6212503263524413,0.5893011087345269,mean,exp/integrated/final/ecom-offers/baseline_plr-evaluation/2.toml,exp/integrated/final/ecom-offers/baseline_plr-evaluation/2,False,321 +final,ecom-offers,baseline_plr,1,AUROC,roc-auc,higher,0.6203331748306973,0.6203331748306973,0.5897210540664094,mean,exp/integrated/final/ecom-offers/baseline_plr-evaluation/1.toml,exp/integrated/final/ecom-offers/baseline_plr-evaluation/1,False,428 +final,ecom-offers,best_rla_only,0,AUROC,roc-auc,higher,0.6227001707049361,0.6227001707049361,0.5911765632808944,mean,exp/integrated/final/ecom-offers/best_rla_only-evaluation/0.toml,exp/integrated/final/ecom-offers/best_rla_only-evaluation/0,False,214 +final,ecom-offers,best_rla_only,1,AUROC,roc-auc,higher,0.6203498803314189,0.6203498803314189,0.5897023859034299,mean,exp/integrated/final/ecom-offers/best_rla_only-evaluation/1.toml,exp/integrated/final/ecom-offers/best_rla_only-evaluation/1,False,428 +final,ecom-offers,best_rla_only,2,AUROC,roc-auc,higher,0.6211526583224122,0.6211526583224122,0.5892307660549951,mean,exp/integrated/final/ecom-offers/best_rla_only-evaluation/2.toml,exp/integrated/final/ecom-offers/best_rla_only-evaluation/2,False,321 +final,ecom-offers,best_esam_only,0,AUROC,roc-auc,higher,0.6227874903903754,0.6227874903903754,0.5911993816464689,mean,exp/integrated/final/ecom-offers/best_esam_only-evaluation/0.toml,exp/integrated/final/ecom-offers/best_esam_only-evaluation/0,False,321 +final,ecom-offers,best_esam_only,1,AUROC,roc-auc,higher,0.6204141557865444,0.6204141557865444,0.5897760078319128,mean,exp/integrated/final/ecom-offers/best_esam_only-evaluation/1.toml,exp/integrated/final/ecom-offers/best_esam_only-evaluation/1,False,428 +final,ecom-offers,best_esam_only,2,AUROC,roc-auc,higher,0.6211550537725488,0.6211550537725488,0.5892400551940371,mean,exp/integrated/final/ecom-offers/best_esam_only-evaluation/2.toml,exp/integrated/final/ecom-offers/best_esam_only-evaluation/2,False,321 +final,ecom-offers,best_mfb_only,0,AUROC,roc-auc,higher,0.6291199291618959,0.6291199291618959,0.5961977102213372,mean,exp/integrated/final/ecom-offers/best_mfb_only-evaluation/0.toml,exp/integrated/final/ecom-offers/best_mfb_only-evaluation/0,False,428 +final,ecom-offers,best_mfb_only,1,AUROC,roc-auc,higher,0.6253906113125361,0.6253906113125361,0.5930931510151916,mean,exp/integrated/final/ecom-offers/best_mfb_only-evaluation/1.toml,exp/integrated/final/ecom-offers/best_mfb_only-evaluation/1,False,428 +final,ecom-offers,best_mfb_only,2,AUROC,roc-auc,higher,0.6220034632459894,0.6220034632459894,0.5900390978219552,mean,exp/integrated/final/ecom-offers/best_mfb_only-evaluation/2.toml,exp/integrated/final/ecom-offers/best_mfb_only-evaluation/2,False,321 +final,ecom-offers,best_cf_fisd_only,0,AUROC,roc-auc,higher,0.6228186386127663,0.6228186386127663,0.5911736900637132,mean,exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/0.toml,exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/0,False,321 +final,ecom-offers,best_cf_fisd_only,1,AUROC,roc-auc,higher,0.6202479078617598,0.6202479078617598,0.5895844228260869,mean,exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/1.toml,exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/1,False,428 +final,ecom-offers,best_cf_fisd_only,2,AUROC,roc-auc,higher,0.6211436845976699,0.6211436845976699,0.5891830464738508,mean,exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/2,False,321 +final,ecom-offers,best_combined,0,AUROC,roc-auc,higher,0.6250550271749447,0.6250550271749447,0.592670158895295,mean,exp/integrated/final/ecom-offers/best_combined-evaluation/0.toml,exp/integrated/final/ecom-offers/best_combined-evaluation/0,False,321 +final,ecom-offers,best_combined,1,AUROC,roc-auc,higher,0.6213201555665758,0.6213201555665758,0.5900624771932574,mean,exp/integrated/final/ecom-offers/best_combined-evaluation/1.toml,exp/integrated/final/ecom-offers/best_combined-evaluation/1,False,428 +final,ecom-offers,best_combined,2,AUROC,roc-auc,higher,0.6208476032754839,0.6208476032754839,0.5890089208341323,mean,exp/integrated/final/ecom-offers/best_combined-evaluation/2.toml,exp/integrated/final/ecom-offers/best_combined-evaluation/2,False,321 +final,ecom-offers,baseline_plr,0,AUROC,roc-auc,higher,0.6404461738890127,0.6404461738890127,0.6034781946000056,best-head,exp/integrated/final/ecom-offers/baseline_plr-evaluation/0.toml,exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/0,False,321 +final,ecom-offers,baseline_plr,2,AUROC,roc-auc,higher,0.6342271136151092,0.6342271136151092,0.5992387053088735,best-head,exp/integrated/final/ecom-offers/baseline_plr-evaluation/2.toml,exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/2,False,321 +final,ecom-offers,baseline_plr,1,AUROC,roc-auc,higher,0.6313621036574809,0.6313621036574809,0.5963646574673217,best-head,exp/integrated/final/ecom-offers/baseline_plr-evaluation/1.toml,exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/1,False,428 +final,ecom-offers,baseline_plr,0,AUROC,roc-auc,higher,0.6413971749638401,0.6413971749638401,0.6040305589800323,greedy-heads,exp/integrated/final/ecom-offers/baseline_plr-evaluation/0.toml,exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/0,False,321 +final,ecom-offers,baseline_plr,2,AUROC,roc-auc,higher,0.6356383175142475,0.6356383175142475,0.5998927728985708,greedy-heads,exp/integrated/final/ecom-offers/baseline_plr-evaluation/2.toml,exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/2,False,321 +final,ecom-offers,baseline_plr,1,AUROC,roc-auc,higher,0.6327357100453991,0.6327357100453991,0.5971884680332056,greedy-heads,exp/integrated/final/ecom-offers/baseline_plr-evaluation/1.toml,exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/1,False,428 +final,ecom-offers,best_rla_only,0,AUROC,roc-auc,higher,0.6396541901648656,0.6396541901648656,0.6015418321886687,best-head,exp/integrated/final/ecom-offers/best_rla_only-evaluation/0.toml,exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/0,False,214 +final,ecom-offers,best_rla_only,2,AUROC,roc-auc,higher,0.633681175787757,0.633681175787757,0.597700306723218,best-head,exp/integrated/final/ecom-offers/best_rla_only-evaluation/2.toml,exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/2,False,321 +final,ecom-offers,best_rla_only,1,AUROC,roc-auc,higher,0.6311484553024557,0.6311484553024557,0.5960986415689524,best-head,exp/integrated/final/ecom-offers/best_rla_only-evaluation/1.toml,exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/1,False,428 +final,ecom-offers,best_rla_only,0,AUROC,roc-auc,higher,0.6396541901648656,0.6396541901648656,0.6015418321886687,greedy-heads,exp/integrated/final/ecom-offers/best_rla_only-evaluation/0.toml,exp/integrated/final/ecom-offers/best_rla_only-greedy-heads-evaluation/0,False,214 +final,ecom-offers,best_rla_only,2,AUROC,roc-auc,higher,0.6351641473729864,0.6351641473729864,0.5998596611627065,greedy-heads,exp/integrated/final/ecom-offers/best_rla_only-evaluation/2.toml,exp/integrated/final/ecom-offers/best_rla_only-greedy-heads-evaluation/2,False,321 +final,ecom-offers,best_rla_only,1,AUROC,roc-auc,higher,0.6323603724914644,0.6323603724914644,0.5968416363152236,greedy-heads,exp/integrated/final/ecom-offers/best_rla_only-evaluation/1.toml,exp/integrated/final/ecom-offers/best_rla_only-greedy-heads-evaluation/1,False,428 +final,ecom-offers,best_esam_only,0,AUROC,roc-auc,higher,0.6410241296714208,0.6410241296714208,0.6024966050474919,best-head,exp/integrated/final/ecom-offers/best_esam_only-evaluation/0.toml,exp/integrated/final/ecom-offers/best_esam_only-best-head-evaluation/0,False,321 +final,ecom-offers,best_esam_only,1,AUROC,roc-auc,higher,0.6315182980623079,0.6315182980623079,0.5962485571769538,best-head,exp/integrated/final/ecom-offers/best_esam_only-evaluation/1.toml,exp/integrated/final/ecom-offers/best_esam_only-best-head-evaluation/1,False,428 +final,ecom-offers,best_esam_only,2,AUROC,roc-auc,higher,0.6341914987995403,0.6341914987995403,0.599267480873393,best-head,exp/integrated/final/ecom-offers/best_esam_only-evaluation/2.toml,exp/integrated/final/ecom-offers/best_esam_only-best-head-evaluation/2,False,321 +final,ecom-offers,best_esam_only,0,AUROC,roc-auc,higher,0.641845068859757,0.641845068859757,0.6041622806453656,greedy-heads,exp/integrated/final/ecom-offers/best_esam_only-evaluation/0.toml,exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/0,False,321 +final,ecom-offers,best_esam_only,2,AUROC,roc-auc,higher,0.6354981541987959,0.6354981541987959,0.5998512677731419,greedy-heads,exp/integrated/final/ecom-offers/best_esam_only-evaluation/2.toml,exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/2,False,321 +final,ecom-offers,best_esam_only,1,AUROC,roc-auc,higher,0.6326258289049813,0.6326258289049813,0.5972452690882881,greedy-heads,exp/integrated/final/ecom-offers/best_esam_only-evaluation/1.toml,exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/1,False,428 +final,ecom-offers,best_mfb_only,0,AUROC,roc-auc,higher,0.6547032702847417,0.6547032702847417,0.6118564029158264,best-head,exp/integrated/final/ecom-offers/best_mfb_only-evaluation/0.toml,exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/0,False,428 +final,ecom-offers,best_mfb_only,2,AUROC,roc-auc,higher,0.6546592087434608,0.6546592087434608,0.6088559366177836,best-head,exp/integrated/final/ecom-offers/best_mfb_only-evaluation/2.toml,exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/2,False,321 +final,ecom-offers,best_mfb_only,1,AUROC,roc-auc,higher,0.6525683493050402,0.6525683493050402,0.6078370975247275,best-head,exp/integrated/final/ecom-offers/best_mfb_only-evaluation/1.toml,exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/1,False,428 +final,ecom-offers,best_mfb_only,0,AUROC,roc-auc,higher,0.6547032702847417,0.6547032702847417,0.6118564029158264,greedy-heads,exp/integrated/final/ecom-offers/best_mfb_only-evaluation/0.toml,exp/integrated/final/ecom-offers/best_mfb_only-greedy-heads-evaluation/0,False,428 +final,ecom-offers,best_mfb_only,2,AUROC,roc-auc,higher,0.6546592087434608,0.6546592087434608,0.6088559366177836,greedy-heads,exp/integrated/final/ecom-offers/best_mfb_only-evaluation/2.toml,exp/integrated/final/ecom-offers/best_mfb_only-greedy-heads-evaluation/2,False,321 +final,ecom-offers,best_mfb_only,1,AUROC,roc-auc,higher,0.6527163070467052,0.6527163070467052,0.6065300223672054,greedy-heads,exp/integrated/final/ecom-offers/best_mfb_only-evaluation/1.toml,exp/integrated/final/ecom-offers/best_mfb_only-greedy-heads-evaluation/1,False,428 +final,ecom-offers,best_cf_fisd_only,0,AUROC,roc-auc,higher,0.6410202011331968,0.6410202011331968,0.6024701571918213,best-head,exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/0.toml,exp/integrated/final/ecom-offers/best_cf_fisd_only-best-head-evaluation/0,False,321 +final,ecom-offers,best_cf_fisd_only,2,AUROC,roc-auc,higher,0.6333770862914994,0.6333770862914994,0.5972389647476771,best-head,exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/ecom-offers/best_cf_fisd_only-best-head-evaluation/2,False,321 +final,ecom-offers,best_cf_fisd_only,1,AUROC,roc-auc,higher,0.6322301668779651,0.6322301668779651,0.5946009422441442,best-head,exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/1.toml,exp/integrated/final/ecom-offers/best_cf_fisd_only-best-head-evaluation/1,False,428 +final,ecom-offers,best_cf_fisd_only,0,AUROC,roc-auc,higher,0.6419149054445072,0.6419149054445072,0.6041224275423289,greedy-heads,exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/0.toml,exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/0,False,321 +final,ecom-offers,best_cf_fisd_only,2,AUROC,roc-auc,higher,0.6340638176219543,0.6340638176219543,0.598916340879379,greedy-heads,exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/2,False,321 +final,ecom-offers,best_cf_fisd_only,1,AUROC,roc-auc,higher,0.6333421643138163,0.6333421643138163,0.5975759215250558,greedy-heads,exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/1.toml,exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/1,False,428 +final,ecom-offers,best_combined,0,AUROC,roc-auc,higher,0.6477890245840551,0.6477890245840551,0.6045088303107439,best-head,exp/integrated/final/ecom-offers/best_combined-evaluation/0.toml,exp/integrated/final/ecom-offers/best_combined-best-head-evaluation/0,False,321 +final,ecom-offers,best_combined,2,AUROC,roc-auc,higher,0.6430199007953441,0.6430199007953441,0.5970664756329549,best-head,exp/integrated/final/ecom-offers/best_combined-evaluation/2.toml,exp/integrated/final/ecom-offers/best_combined-best-head-evaluation/2,False,321 +final,ecom-offers,best_combined,1,AUROC,roc-auc,higher,0.6512227217747169,0.6512227217747169,0.6075381341628896,best-head,exp/integrated/final/ecom-offers/best_combined-evaluation/1.toml,exp/integrated/final/ecom-offers/best_combined-best-head-evaluation/1,False,428 +final,ecom-offers,best_combined,0,AUROC,roc-auc,higher,0.6506783465179156,0.6506783465179156,0.6065618416202598,greedy-heads,exp/integrated/final/ecom-offers/best_combined-evaluation/0.toml,exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/0,False,321 +final,ecom-offers,best_combined,1,AUROC,roc-auc,higher,0.6512227217747169,0.6512227217747169,0.6075381341628896,greedy-heads,exp/integrated/final/ecom-offers/best_combined-evaluation/1.toml,exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/1,False,428 +final,ecom-offers,best_combined,2,AUROC,roc-auc,higher,0.6436592869751769,0.6436592869751769,0.6015328994897303,greedy-heads,exp/integrated/final/ecom-offers/best_combined-evaluation/2.toml,exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/2,False,321 +final,homesite-insurance,baseline_plr,0,AUROC,roc-auc,higher,0.9583121450834431,0.9583121450834431,0.9626260391126089,mean,exp/integrated/final/homesite-insurance/baseline_plr-evaluation/0.toml,exp/integrated/final/homesite-insurance/baseline_plr-evaluation/0,False,1980 +final,homesite-insurance,baseline_plr,1,AUROC,roc-auc,higher,0.9586997364143323,0.9586997364143323,0.9622916084556132,mean,exp/integrated/final/homesite-insurance/baseline_plr-evaluation/1.toml,exp/integrated/final/homesite-insurance/baseline_plr-evaluation/1,False,1760 +final,homesite-insurance,baseline_plr,2,AUROC,roc-auc,higher,0.9582475934195426,0.9582475934195426,0.9625649084455907,mean,exp/integrated/final/homesite-insurance/baseline_plr-evaluation/2.toml,exp/integrated/final/homesite-insurance/baseline_plr-evaluation/2,False,1980 +final,homesite-insurance,best_rla_only,0,AUROC,roc-auc,higher,0.9581626616470993,0.9581626616470993,0.9620964494052155,mean,exp/integrated/final/homesite-insurance/best_rla_only-evaluation/0.toml,exp/integrated/final/homesite-insurance/best_rla_only-evaluation/0,False,1760 +final,homesite-insurance,best_rla_only,1,AUROC,roc-auc,higher,0.9583407546059616,0.9583407546059616,0.9620763221935056,mean,exp/integrated/final/homesite-insurance/best_rla_only-evaluation/1.toml,exp/integrated/final/homesite-insurance/best_rla_only-evaluation/1,False,1760 +final,homesite-insurance,best_rla_only,2,AUROC,roc-auc,higher,0.9581674973070209,0.9581674973070209,0.9627480104988175,mean,exp/integrated/final/homesite-insurance/best_rla_only-evaluation/2.toml,exp/integrated/final/homesite-insurance/best_rla_only-evaluation/2,False,1980 +final,homesite-insurance,best_esam_only,0,AUROC,roc-auc,higher,0.9583120395781356,0.9583120395781356,0.9627483366901237,mean,exp/integrated/final/homesite-insurance/best_esam_only-evaluation/0.toml,exp/integrated/final/homesite-insurance/best_esam_only-evaluation/0,False,1980 +final,homesite-insurance,best_esam_only,1,AUROC,roc-auc,higher,0.9584753793781823,0.9584753793781823,0.9624917932683585,mean,exp/integrated/final/homesite-insurance/best_esam_only-evaluation/1.toml,exp/integrated/final/homesite-insurance/best_esam_only-evaluation/1,False,1760 +final,homesite-insurance,best_esam_only,2,AUROC,roc-auc,higher,0.9582681142018287,0.9582681142018287,0.9627356393915001,mean,exp/integrated/final/homesite-insurance/best_esam_only-evaluation/2.toml,exp/integrated/final/homesite-insurance/best_esam_only-evaluation/2,False,1980 +final,homesite-insurance,best_mfb_only,0,AUROC,roc-auc,higher,0.9584246137411132,0.9584246137411132,0.962318960200698,mean,exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/0.toml,exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/0,False,1980 +final,homesite-insurance,best_mfb_only,1,AUROC,roc-auc,higher,0.9585282902898349,0.9585282902898349,0.962430602195543,mean,exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/1.toml,exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/1,False,1980 +final,homesite-insurance,best_mfb_only,2,AUROC,roc-auc,higher,0.9583243485306635,0.9583243485306635,0.9625461826483815,mean,exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/2.toml,exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/2,False,1980 +final,homesite-insurance,best_cf_fisd_only,0,AUROC,roc-auc,higher,0.9589599740471012,0.9589599740471012,0.9628908822909436,mean,exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/0.toml,exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/0,False,2420 +final,homesite-insurance,best_cf_fisd_only,1,AUROC,roc-auc,higher,0.9589376508824807,0.9589376508824807,0.9627469836002609,mean,exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/1.toml,exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/1,False,1760 +final,homesite-insurance,best_cf_fisd_only,2,AUROC,roc-auc,higher,0.9589488871977169,0.9589488871977169,0.9624131449200803,mean,exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/2,False,1980 +final,homesite-insurance,best_combined,0,AUROC,roc-auc,higher,0.9589238560635404,0.9589238560635404,0.9624954901031625,mean,exp/integrated/final/homesite-insurance/best_combined-evaluation/0.toml,exp/integrated/final/homesite-insurance/best_combined-evaluation/0,False,1980 +final,homesite-insurance,best_combined,1,AUROC,roc-auc,higher,0.9587836483021374,0.9587836483021374,0.9621208050227469,mean,exp/integrated/final/homesite-insurance/best_combined-evaluation/1.toml,exp/integrated/final/homesite-insurance/best_combined-evaluation/1,False,1760 +final,homesite-insurance,best_combined,2,AUROC,roc-auc,higher,0.9585690329227027,0.9585690329227027,0.9620638665180715,mean,exp/integrated/final/homesite-insurance/best_combined-evaluation/2.toml,exp/integrated/final/homesite-insurance/best_combined-evaluation/2,False,2200 +final,homesite-insurance,baseline_plr,0,AUROC,roc-auc,higher,0.9554763470552776,0.9554763470552776,0.959416932798502,best-head,exp/integrated/final/homesite-insurance/baseline_plr-evaluation/0.toml,exp/integrated/final/homesite-insurance/baseline_plr-best-head-evaluation/0,False,1980 +final,homesite-insurance,baseline_plr,1,AUROC,roc-auc,higher,0.9563250405399748,0.9563250405399748,0.9592494275221765,best-head,exp/integrated/final/homesite-insurance/baseline_plr-evaluation/1.toml,exp/integrated/final/homesite-insurance/baseline_plr-best-head-evaluation/1,False,1760 +final,homesite-insurance,baseline_plr,2,AUROC,roc-auc,higher,0.9561054927874199,0.9561054927874199,0.9608390423255206,best-head,exp/integrated/final/homesite-insurance/baseline_plr-evaluation/2.toml,exp/integrated/final/homesite-insurance/baseline_plr-best-head-evaluation/2,False,1980 +final,homesite-insurance,baseline_plr,0,AUROC,roc-auc,higher,0.9589474980445032,0.9589474980445032,0.9625170670540112,greedy-heads,exp/integrated/final/homesite-insurance/baseline_plr-evaluation/0.toml,exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/0,False,1980 +final,homesite-insurance,baseline_plr,1,AUROC,roc-auc,higher,0.9593694489371631,0.9593694489371631,0.9625145058481993,greedy-heads,exp/integrated/final/homesite-insurance/baseline_plr-evaluation/1.toml,exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/1,False,1760 +final,homesite-insurance,baseline_plr,2,AUROC,roc-auc,higher,0.9591019138539669,0.9591019138539669,0.9629950822915426,greedy-heads,exp/integrated/final/homesite-insurance/baseline_plr-evaluation/2.toml,exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/2,False,1980 +final,homesite-insurance,best_rla_only,0,AUROC,roc-auc,higher,0.9560453459701026,0.9560453459701026,0.9600924508313724,best-head,exp/integrated/final/homesite-insurance/best_rla_only-evaluation/0.toml,exp/integrated/final/homesite-insurance/best_rla_only-best-head-evaluation/0,False,1760 +final,homesite-insurance,best_rla_only,1,AUROC,roc-auc,higher,0.9560549117846388,0.9560549117846388,0.9591263205069757,best-head,exp/integrated/final/homesite-insurance/best_rla_only-evaluation/1.toml,exp/integrated/final/homesite-insurance/best_rla_only-best-head-evaluation/1,False,1760 +final,homesite-insurance,best_rla_only,2,AUROC,roc-auc,higher,0.9556372338569274,0.9556372338569274,0.9608528752531366,best-head,exp/integrated/final/homesite-insurance/best_rla_only-evaluation/2.toml,exp/integrated/final/homesite-insurance/best_rla_only-best-head-evaluation/2,False,1980 +final,homesite-insurance,best_rla_only,0,AUROC,roc-auc,higher,0.9591329763748824,0.9591329763748824,0.962370812537228,greedy-heads,exp/integrated/final/homesite-insurance/best_rla_only-evaluation/0.toml,exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/0,False,1760 +final,homesite-insurance,best_rla_only,1,AUROC,roc-auc,higher,0.9590165424760763,0.9590165424760763,0.9620755973239362,greedy-heads,exp/integrated/final/homesite-insurance/best_rla_only-evaluation/1.toml,exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/1,False,1760 +final,homesite-insurance,best_rla_only,2,AUROC,roc-auc,higher,0.9587859694188998,0.9587859694188998,0.9632457301074769,greedy-heads,exp/integrated/final/homesite-insurance/best_rla_only-evaluation/2.toml,exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/2,False,1980 +final,homesite-insurance,best_esam_only,0,AUROC,roc-auc,higher,0.9561512908829329,0.9561512908829329,0.9606910602029306,best-head,exp/integrated/final/homesite-insurance/best_esam_only-evaluation/0.toml,exp/integrated/final/homesite-insurance/best_esam_only-best-head-evaluation/0,False,1980 +final,homesite-insurance,best_esam_only,1,AUROC,roc-auc,higher,0.9562486459053205,0.9562486459053205,0.9594156280332771,best-head,exp/integrated/final/homesite-insurance/best_esam_only-evaluation/1.toml,exp/integrated/final/homesite-insurance/best_esam_only-best-head-evaluation/1,False,1760 +final,homesite-insurance,best_esam_only,2,AUROC,roc-auc,higher,0.9550014764588559,0.9550014764588559,0.9593146174587831,best-head,exp/integrated/final/homesite-insurance/best_esam_only-evaluation/2.toml,exp/integrated/final/homesite-insurance/best_esam_only-best-head-evaluation/2,False,1980 +final,homesite-insurance,best_esam_only,0,AUROC,roc-auc,higher,0.958991133281215,0.958991133281215,0.9627938464179217,greedy-heads,exp/integrated/final/homesite-insurance/best_esam_only-evaluation/0.toml,exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/0,False,1980 +final,homesite-insurance,best_esam_only,1,AUROC,roc-auc,higher,0.9591595812965608,0.9591595812965608,0.9622548696496048,greedy-heads,exp/integrated/final/homesite-insurance/best_esam_only-evaluation/1.toml,exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/1,False,1760 +final,homesite-insurance,best_esam_only,2,AUROC,roc-auc,higher,0.9590920051471822,0.9590920051471822,0.9631468941416912,greedy-heads,exp/integrated/final/homesite-insurance/best_esam_only-evaluation/2.toml,exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/2,False,1980 +final,homesite-insurance,best_mfb_only,0,AUROC,roc-auc,higher,0.9563975490624737,0.9563975490624737,0.9593891219693567,best-head,exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/0.toml,exp/integrated/final/homesite-insurance/best_mfb_only-best-head-evaluation/0,False,1980 +final,homesite-insurance,best_mfb_only,1,AUROC,roc-auc,higher,0.9550071913296724,0.9550071913296724,0.9596843251014889,best-head,exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/1.toml,exp/integrated/final/homesite-insurance/best_mfb_only-best-head-evaluation/1,False,1980 +final,homesite-insurance,best_mfb_only,2,AUROC,roc-auc,higher,0.9545391258255768,0.9545391258255768,0.9588200872762291,best-head,exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/2.toml,exp/integrated/final/homesite-insurance/best_mfb_only-best-head-evaluation/2,False,1980 +final,homesite-insurance,best_mfb_only,0,AUROC,roc-auc,higher,0.9592957095194116,0.9592957095194116,0.9620218603265258,greedy-heads,exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/0.toml,exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/0,False,1980 +final,homesite-insurance,best_mfb_only,1,AUROC,roc-auc,higher,0.9591725848256957,0.9591725848256957,0.9622278199335073,greedy-heads,exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/1.toml,exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/1,False,1980 +final,homesite-insurance,best_mfb_only,2,AUROC,roc-auc,higher,0.9590785004678369,0.9590785004678369,0.962897309467792,greedy-heads,exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/2.toml,exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/2,False,1980 +final,homesite-insurance,best_cf_fisd_only,0,AUROC,roc-auc,higher,0.957813544584969,0.957813544584969,0.961490313471295,best-head,exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/0.toml,exp/integrated/final/homesite-insurance/best_cf_fisd_only-best-head-evaluation/0,False,2420 +final,homesite-insurance,best_cf_fisd_only,1,AUROC,roc-auc,higher,0.9580043948939159,0.9580043948939159,0.9620360194454477,best-head,exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/1.toml,exp/integrated/final/homesite-insurance/best_cf_fisd_only-best-head-evaluation/1,False,1760 +final,homesite-insurance,best_cf_fisd_only,2,AUROC,roc-auc,higher,0.9575815999587967,0.9575815999587967,0.9611081622343147,best-head,exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/homesite-insurance/best_cf_fisd_only-best-head-evaluation/2,False,1980 +final,homesite-insurance,best_cf_fisd_only,0,AUROC,roc-auc,higher,0.9594844321379927,0.9594844321379927,0.9634382071404677,greedy-heads,exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/0.toml,exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/0,False,2420 +final,homesite-insurance,best_cf_fisd_only,1,AUROC,roc-auc,higher,0.9596519921503349,0.9596519921503349,0.9627588835423583,greedy-heads,exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/1.toml,exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/1,False,1760 +final,homesite-insurance,best_cf_fisd_only,2,AUROC,roc-auc,higher,0.9594475404488445,0.9594475404488445,0.9625258379758007,greedy-heads,exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/2,False,1980 +final,homesite-insurance,best_combined,0,AUROC,roc-auc,higher,0.9576967765860231,0.9576967765860231,0.9617315500639867,best-head,exp/integrated/final/homesite-insurance/best_combined-evaluation/0.toml,exp/integrated/final/homesite-insurance/best_combined-best-head-evaluation/0,False,1980 +final,homesite-insurance,best_combined,1,AUROC,roc-auc,higher,0.9574705028701225,0.9574705028701225,0.9608438143835193,best-head,exp/integrated/final/homesite-insurance/best_combined-evaluation/1.toml,exp/integrated/final/homesite-insurance/best_combined-best-head-evaluation/1,False,1760 +final,homesite-insurance,best_combined,2,AUROC,roc-auc,higher,0.9571173941905297,0.9571173941905297,0.9613875873721536,best-head,exp/integrated/final/homesite-insurance/best_combined-evaluation/2.toml,exp/integrated/final/homesite-insurance/best_combined-best-head-evaluation/2,False,2200 +final,homesite-insurance,best_combined,0,AUROC,roc-auc,higher,0.9597570226838343,0.9597570226838343,0.9633062567165207,greedy-heads,exp/integrated/final/homesite-insurance/best_combined-evaluation/0.toml,exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/0,False,1980 +final,homesite-insurance,best_combined,1,AUROC,roc-auc,higher,0.9595148440428459,0.9595148440428459,0.9624332721317901,greedy-heads,exp/integrated/final/homesite-insurance/best_combined-evaluation/1.toml,exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/1,False,1760 +final,homesite-insurance,best_combined,2,AUROC,roc-auc,higher,0.9596270313530297,0.9596270313530297,0.9628075343716237,greedy-heads,exp/integrated/final/homesite-insurance/best_combined-evaluation/2.toml,exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/2,False,2200 +final,cooking-time,baseline_plr,0,RMSE,rmse,lower,0.4627596029926704,-0.4627596029926704,0.4802386655626689,mean,exp/integrated/final/cooking-time/baseline_plr-evaluation/0.toml,exp/integrated/final/cooking-time/baseline_plr-evaluation/0,False,23088 +final,cooking-time,baseline_plr,1,RMSE,rmse,lower,0.46274572430329897,-0.46274572430329897,0.4804413950506519,mean,exp/integrated/final/cooking-time/baseline_plr-evaluation/1.toml,exp/integrated/final/cooking-time/baseline_plr-evaluation/1,False,19092 +final,cooking-time,baseline_plr,2,RMSE,rmse,lower,0.4626209262038764,-0.4626209262038764,0.48046188039443216,mean,exp/integrated/final/cooking-time/baseline_plr-evaluation/2.toml,exp/integrated/final/cooking-time/baseline_plr-evaluation/2,False,28638 +final,cooking-time,best_rla_only,0,RMSE,rmse,lower,0.46272482498106277,-0.46272482498106277,0.4802920784266626,mean,exp/integrated/final/cooking-time/best_rla_only-evaluation/0.toml,exp/integrated/final/cooking-time/best_rla_only-evaluation/0,False,23088 +final,cooking-time,best_rla_only,1,RMSE,rmse,lower,0.4627833825636106,-0.4627833825636106,0.4804816670736117,mean,exp/integrated/final/cooking-time/best_rla_only-evaluation/1.toml,exp/integrated/final/cooking-time/best_rla_only-evaluation/1,False,19092 +final,cooking-time,best_rla_only,2,RMSE,rmse,lower,0.4626829107835815,-0.4626829107835815,0.4805534877331418,mean,exp/integrated/final/cooking-time/best_rla_only-evaluation/2.toml,exp/integrated/final/cooking-time/best_rla_only-evaluation/2,False,28638 +final,cooking-time,best_esam_only,0,RMSE,rmse,lower,0.4627563346148938,-0.4627563346148938,0.4802735094812688,mean,exp/integrated/final/cooking-time/best_esam_only-evaluation/0.toml,exp/integrated/final/cooking-time/best_esam_only-evaluation/0,False,23088 +final,cooking-time,best_esam_only,1,RMSE,rmse,lower,0.4627833664641114,-0.4627833664641114,0.4804732935000574,mean,exp/integrated/final/cooking-time/best_esam_only-evaluation/1.toml,exp/integrated/final/cooking-time/best_esam_only-evaluation/1,False,19092 +final,cooking-time,best_esam_only,2,RMSE,rmse,lower,0.4626873390860138,-0.4626873390860138,0.4802976784438942,mean,exp/integrated/final/cooking-time/best_esam_only-evaluation/2.toml,exp/integrated/final/cooking-time/best_esam_only-evaluation/2,False,29526 +final,cooking-time,best_mfb_only,0,RMSE,rmse,lower,0.4634085546239816,-0.4634085546239816,0.4808915146871742,mean,exp/integrated/final/cooking-time/best_mfb_only-evaluation/0.toml,exp/integrated/final/cooking-time/best_mfb_only-evaluation/0,False,23088 +final,cooking-time,best_mfb_only,1,RMSE,rmse,lower,0.4632980549483924,-0.4632980549483924,0.48077915988352504,mean,exp/integrated/final/cooking-time/best_mfb_only-evaluation/1.toml,exp/integrated/final/cooking-time/best_mfb_only-evaluation/1,False,18870 +final,cooking-time,best_mfb_only,2,RMSE,rmse,lower,0.4630663413507467,-0.4630663413507467,0.48078120546833536,mean,exp/integrated/final/cooking-time/best_mfb_only-evaluation/2.toml,exp/integrated/final/cooking-time/best_mfb_only-evaluation/2,False,28638 +final,cooking-time,best_cf_fisd_only,0,RMSE,rmse,lower,0.4632019090821483,-0.4632019090821483,0.4808496810468957,mean,exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/0.toml,exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/0,False,17982 +final,cooking-time,best_cf_fisd_only,1,RMSE,rmse,lower,0.4632797054631699,-0.4632797054631699,0.48088074674610853,mean,exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/1.toml,exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/1,False,14652 +final,cooking-time,best_cf_fisd_only,2,RMSE,rmse,lower,0.46327719662519024,-0.46327719662519024,0.48070024325509664,mean,exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/2,False,12876 +final,cooking-time,best_combined,0,RMSE,rmse,lower,0.4627353230636857,-0.4627353230636857,0.4803077303885999,mean,exp/integrated/final/cooking-time/best_combined-evaluation/0.toml,exp/integrated/final/cooking-time/best_combined-evaluation/0,False,23088 +final,cooking-time,best_combined,1,RMSE,rmse,lower,0.4627920762114039,-0.4627920762114039,0.4804951265119692,mean,exp/integrated/final/cooking-time/best_combined-evaluation/1.toml,exp/integrated/final/cooking-time/best_combined-evaluation/1,False,19092 +final,cooking-time,best_combined,2,RMSE,rmse,lower,0.4627156309124574,-0.4627156309124574,0.4803808798726675,mean,exp/integrated/final/cooking-time/best_combined-evaluation/2.toml,exp/integrated/final/cooking-time/best_combined-evaluation/2,False,29526 +final,cooking-time,baseline_plr,0,RMSE,rmse,lower,0.4652274815891859,-0.4652274815891859,0.4820660657381116,best-head,exp/integrated/final/cooking-time/baseline_plr-evaluation/0.toml,exp/integrated/final/cooking-time/baseline_plr-best-head-evaluation/0,False,23088 +final,cooking-time,baseline_plr,1,RMSE,rmse,lower,0.46461960653787665,-0.46461960653787665,0.4822480664686614,best-head,exp/integrated/final/cooking-time/baseline_plr-evaluation/1.toml,exp/integrated/final/cooking-time/baseline_plr-best-head-evaluation/1,False,19092 +final,cooking-time,baseline_plr,2,RMSE,rmse,lower,0.46567157154030486,-0.46567157154030486,0.48281151175783615,best-head,exp/integrated/final/cooking-time/baseline_plr-evaluation/2.toml,exp/integrated/final/cooking-time/baseline_plr-best-head-evaluation/2,False,28638 +final,cooking-time,baseline_plr,0,RMSE,rmse,lower,0.4625169879783321,-0.4625169879783321,0.47990786370986477,greedy-heads,exp/integrated/final/cooking-time/baseline_plr-evaluation/0.toml,exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/0,False,23088 +final,cooking-time,baseline_plr,1,RMSE,rmse,lower,0.4624852364993238,-0.4624852364993238,0.4800568652741734,greedy-heads,exp/integrated/final/cooking-time/baseline_plr-evaluation/1.toml,exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/1,False,19092 +final,cooking-time,baseline_plr,2,RMSE,rmse,lower,0.46240787039558773,-0.46240787039558773,0.480411417572094,greedy-heads,exp/integrated/final/cooking-time/baseline_plr-evaluation/2.toml,exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/2,False,28638 +final,cooking-time,best_rla_only,0,RMSE,rmse,lower,0.4649795846661889,-0.4649795846661889,0.4827695666979872,best-head,exp/integrated/final/cooking-time/best_rla_only-evaluation/0.toml,exp/integrated/final/cooking-time/best_rla_only-best-head-evaluation/0,False,23088 +final,cooking-time,best_rla_only,1,RMSE,rmse,lower,0.46454139300633795,-0.46454139300633795,0.4826743974666991,best-head,exp/integrated/final/cooking-time/best_rla_only-evaluation/1.toml,exp/integrated/final/cooking-time/best_rla_only-best-head-evaluation/1,False,19092 +final,cooking-time,best_rla_only,2,RMSE,rmse,lower,0.46551838172590837,-0.46551838172590837,0.4845241655350321,best-head,exp/integrated/final/cooking-time/best_rla_only-evaluation/2.toml,exp/integrated/final/cooking-time/best_rla_only-best-head-evaluation/2,False,28638 +final,cooking-time,best_rla_only,0,RMSE,rmse,lower,0.4624107384246987,-0.4624107384246987,0.479974942646056,greedy-heads,exp/integrated/final/cooking-time/best_rla_only-evaluation/0.toml,exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/0,False,23088 +final,cooking-time,best_rla_only,1,RMSE,rmse,lower,0.46252858614858405,-0.46252858614858405,0.48029415711103896,greedy-heads,exp/integrated/final/cooking-time/best_rla_only-evaluation/1.toml,exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/1,False,19092 +final,cooking-time,best_rla_only,2,RMSE,rmse,lower,0.462385779530238,-0.462385779530238,0.4801696683856488,greedy-heads,exp/integrated/final/cooking-time/best_rla_only-evaluation/2.toml,exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/2,False,28638 +final,cooking-time,best_esam_only,0,RMSE,rmse,lower,0.46513258351328474,-0.46513258351328474,0.48253251942068065,best-head,exp/integrated/final/cooking-time/best_esam_only-evaluation/0.toml,exp/integrated/final/cooking-time/best_esam_only-best-head-evaluation/0,False,23088 +final,cooking-time,best_esam_only,1,RMSE,rmse,lower,0.46441974054703444,-0.46441974054703444,0.48248829558115747,best-head,exp/integrated/final/cooking-time/best_esam_only-evaluation/1.toml,exp/integrated/final/cooking-time/best_esam_only-best-head-evaluation/1,False,19092 +final,cooking-time,best_esam_only,2,RMSE,rmse,lower,0.4655328979544962,-0.4655328979544962,0.48239242216697287,best-head,exp/integrated/final/cooking-time/best_esam_only-evaluation/2.toml,exp/integrated/final/cooking-time/best_esam_only-best-head-evaluation/2,False,29526 +final,cooking-time,best_esam_only,0,RMSE,rmse,lower,0.46246732196836127,-0.46246732196836127,0.4800169922135662,greedy-heads,exp/integrated/final/cooking-time/best_esam_only-evaluation/0.toml,exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/0,False,23088 +final,cooking-time,best_esam_only,1,RMSE,rmse,lower,0.46256428091435436,-0.46256428091435436,0.4801574722160279,greedy-heads,exp/integrated/final/cooking-time/best_esam_only-evaluation/1.toml,exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/1,False,19092 +final,cooking-time,best_esam_only,2,RMSE,rmse,lower,0.462401054726952,-0.462401054726952,0.48002014306639385,greedy-heads,exp/integrated/final/cooking-time/best_esam_only-evaluation/2.toml,exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/2,False,29526 +final,cooking-time,best_mfb_only,0,RMSE,rmse,lower,0.4652213638497702,-0.4652213638497702,0.48343941871247514,best-head,exp/integrated/final/cooking-time/best_mfb_only-evaluation/0.toml,exp/integrated/final/cooking-time/best_mfb_only-best-head-evaluation/0,False,23088 +final,cooking-time,best_mfb_only,1,RMSE,rmse,lower,0.4651112788369628,-0.4651112788369628,0.4825797035175412,best-head,exp/integrated/final/cooking-time/best_mfb_only-evaluation/1.toml,exp/integrated/final/cooking-time/best_mfb_only-best-head-evaluation/1,False,18870 +final,cooking-time,best_mfb_only,2,RMSE,rmse,lower,0.4653975929915622,-0.4653975929915622,0.48330849475451526,best-head,exp/integrated/final/cooking-time/best_mfb_only-evaluation/2.toml,exp/integrated/final/cooking-time/best_mfb_only-best-head-evaluation/2,False,28638 +final,cooking-time,best_mfb_only,0,RMSE,rmse,lower,0.4624822561623469,-0.4624822561623469,0.4798057758257153,greedy-heads,exp/integrated/final/cooking-time/best_mfb_only-evaluation/0.toml,exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/0,False,23088 +final,cooking-time,best_mfb_only,1,RMSE,rmse,lower,0.4626697544496206,-0.4626697544496206,0.47992630708272743,greedy-heads,exp/integrated/final/cooking-time/best_mfb_only-evaluation/1.toml,exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/1,False,18870 +final,cooking-time,best_mfb_only,2,RMSE,rmse,lower,0.462357531977379,-0.462357531977379,0.48018644149208806,greedy-heads,exp/integrated/final/cooking-time/best_mfb_only-evaluation/2.toml,exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/2,False,28638 +final,cooking-time,best_cf_fisd_only,0,RMSE,rmse,lower,0.4638214410008231,-0.4638214410008231,0.4819659035779245,best-head,exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/0.toml,exp/integrated/final/cooking-time/best_cf_fisd_only-best-head-evaluation/0,False,17982 +final,cooking-time,best_cf_fisd_only,2,RMSE,rmse,lower,0.46399304740420017,-0.46399304740420017,0.4807545037031361,best-head,exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/cooking-time/best_cf_fisd_only-best-head-evaluation/2,False,12876 +final,cooking-time,best_cf_fisd_only,1,RMSE,rmse,lower,0.4640342331314561,-0.4640342331314561,0.4812057394258983,best-head,exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/1.toml,exp/integrated/final/cooking-time/best_cf_fisd_only-best-head-evaluation/1,False,14652 +final,cooking-time,best_cf_fisd_only,0,RMSE,rmse,lower,0.4628482268025946,-0.4628482268025946,0.4803119031248436,greedy-heads,exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/0.toml,exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/0,False,17982 +final,cooking-time,best_cf_fisd_only,2,RMSE,rmse,lower,0.46301623543457804,-0.46301623543457804,0.4803092971079454,greedy-heads,exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/2,False,12876 +final,cooking-time,best_cf_fisd_only,1,RMSE,rmse,lower,0.4629984701854322,-0.4629984701854322,0.4803911006799199,greedy-heads,exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/1.toml,exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/1,False,14652 +final,cooking-time,best_combined,0,RMSE,rmse,lower,0.46492255368357066,-0.46492255368357066,0.4825495500767423,best-head,exp/integrated/final/cooking-time/best_combined-evaluation/0.toml,exp/integrated/final/cooking-time/best_combined-best-head-evaluation/0,False,23088 +final,cooking-time,best_combined,1,RMSE,rmse,lower,0.46452431161307095,-0.46452431161307095,0.48248902135425475,best-head,exp/integrated/final/cooking-time/best_combined-evaluation/1.toml,exp/integrated/final/cooking-time/best_combined-best-head-evaluation/1,False,19092 +final,cooking-time,best_combined,2,RMSE,rmse,lower,0.46544124766603506,-0.46544124766603506,0.4832114117959526,best-head,exp/integrated/final/cooking-time/best_combined-evaluation/2.toml,exp/integrated/final/cooking-time/best_combined-best-head-evaluation/2,False,29526 +final,cooking-time,best_combined,0,RMSE,rmse,lower,0.4624234187671886,-0.4624234187671886,0.4800586966547807,greedy-heads,exp/integrated/final/cooking-time/best_combined-evaluation/0.toml,exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/0,False,23088 +final,cooking-time,best_combined,1,RMSE,rmse,lower,0.46250972286520653,-0.46250972286520653,0.4801020579816987,greedy-heads,exp/integrated/final/cooking-time/best_combined-evaluation/1.toml,exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/1,False,19092 +final,cooking-time,best_combined,2,RMSE,rmse,lower,0.4623945128795896,-0.4623945128795896,0.4799174115040007,greedy-heads,exp/integrated/final/cooking-time/best_combined-evaluation/2.toml,exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/2,False,29526 +final,delivery-eta,baseline_plr,0,RMSE,rmse,lower,0.5558749906446844,-0.5558749906446844,0.5496162918403876,mean,exp/integrated/final/delivery-eta/baseline_plr-evaluation/0.toml,exp/integrated/final/delivery-eta/baseline_plr-evaluation/0,False,2184 +final,delivery-eta,baseline_plr,1,RMSE,rmse,lower,0.5562247345576334,-0.5562247345576334,0.550284872190118,mean,exp/integrated/final/delivery-eta/baseline_plr-evaluation/1.toml,exp/integrated/final/delivery-eta/baseline_plr-evaluation/1,False,1638 +final,delivery-eta,baseline_plr,2,RMSE,rmse,lower,0.5557493871736087,-0.5557493871736087,0.5507763801521746,mean,exp/integrated/final/delivery-eta/baseline_plr-evaluation/2.toml,exp/integrated/final/delivery-eta/baseline_plr-evaluation/2,False,1638 +final,delivery-eta,best_rla_only,0,RMSE,rmse,lower,0.5559356776481553,-0.5559356776481553,0.55135486033622,mean,exp/integrated/final/delivery-eta/best_rla_only-evaluation/0.toml,exp/integrated/final/delivery-eta/best_rla_only-evaluation/0,False,1365 +final,delivery-eta,best_rla_only,1,RMSE,rmse,lower,0.5564338969448529,-0.5564338969448529,0.5491088203688107,mean,exp/integrated/final/delivery-eta/best_rla_only-evaluation/1.toml,exp/integrated/final/delivery-eta/best_rla_only-evaluation/1,False,1365 +final,delivery-eta,best_rla_only,2,RMSE,rmse,lower,0.5562272527951271,-0.5562272527951271,0.5494697322909359,mean,exp/integrated/final/delivery-eta/best_rla_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_rla_only-evaluation/2,False,1911 +final,delivery-eta,best_esam_only,0,RMSE,rmse,lower,0.5558269510040955,-0.5558269510040955,0.5527269612276788,mean,exp/integrated/final/delivery-eta/best_esam_only-evaluation/0.toml,exp/integrated/final/delivery-eta/best_esam_only-evaluation/0,False,2730 +final,delivery-eta,best_esam_only,1,RMSE,rmse,lower,0.5562247345576334,-0.5562247345576334,0.550284872190118,mean,exp/integrated/final/delivery-eta/best_esam_only-evaluation/1.toml,exp/integrated/final/delivery-eta/best_esam_only-evaluation/1,False,1638 +final,delivery-eta,best_esam_only,2,RMSE,rmse,lower,0.5557493871736087,-0.5557493871736087,0.5507763801521746,mean,exp/integrated/final/delivery-eta/best_esam_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_esam_only-evaluation/2,False,1638 +final,delivery-eta,best_mfb_only,1,RMSE,rmse,lower,0.5559520276955289,-0.5559520276955289,0.5503973468399834,mean,exp/integrated/final/delivery-eta/best_mfb_only-evaluation/1.toml,exp/integrated/final/delivery-eta/best_mfb_only-evaluation/1,False,1638 +final,delivery-eta,best_mfb_only,2,RMSE,rmse,lower,0.5562502378827562,-0.5562502378827562,0.5500394546922427,mean,exp/integrated/final/delivery-eta/best_mfb_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_mfb_only-evaluation/2,False,1638 +final,delivery-eta,best_mfb_only,0,RMSE,rmse,lower,0.5558048331442785,-0.5558048331442785,0.5501619198178428,mean,exp/integrated/final/delivery-eta/best_mfb_only-evaluation/0.toml,exp/integrated/final/delivery-eta/best_mfb_only-evaluation/0,False,2184 +final,delivery-eta,best_cf_fisd_only,1,RMSE,rmse,lower,0.5562247345576334,-0.5562247345576334,0.550284872190118,mean,exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/1.toml,exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/1,False,1638 +final,delivery-eta,best_cf_fisd_only,0,RMSE,rmse,lower,0.5558749906446844,-0.5558749906446844,0.5496162918403876,mean,exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/0.toml,exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/0,False,2184 +final,delivery-eta,best_cf_fisd_only,2,RMSE,rmse,lower,0.5557493871736087,-0.5557493871736087,0.5507763801521746,mean,exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/2,False,1638 +final,delivery-eta,best_combined,1,RMSE,rmse,lower,0.5564338969448529,-0.5564338969448529,0.5491088203688107,mean,exp/integrated/final/delivery-eta/best_combined-evaluation/1.toml,exp/integrated/final/delivery-eta/best_combined-evaluation/1,False,1365 +final,delivery-eta,best_combined,0,RMSE,rmse,lower,0.5559356776481553,-0.5559356776481553,0.55135486033622,mean,exp/integrated/final/delivery-eta/best_combined-evaluation/0.toml,exp/integrated/final/delivery-eta/best_combined-evaluation/0,False,1365 +final,delivery-eta,best_combined,2,RMSE,rmse,lower,0.5562272527951271,-0.5562272527951271,0.5494697322909359,mean,exp/integrated/final/delivery-eta/best_combined-evaluation/2.toml,exp/integrated/final/delivery-eta/best_combined-evaluation/2,False,1911 +final,delivery-eta,baseline_plr,0,RMSE,rmse,lower,0.5583881453279098,-0.5583881453279098,0.5557654209580059,best-head,exp/integrated/final/delivery-eta/baseline_plr-evaluation/0.toml,exp/integrated/final/delivery-eta/baseline_plr-best-head-evaluation/0,False,2184 +final,delivery-eta,baseline_plr,1,RMSE,rmse,lower,0.5578575885405295,-0.5578575885405295,0.5568376961917461,best-head,exp/integrated/final/delivery-eta/baseline_plr-evaluation/1.toml,exp/integrated/final/delivery-eta/baseline_plr-best-head-evaluation/1,False,1638 +final,delivery-eta,baseline_plr,2,RMSE,rmse,lower,0.5576450053690213,-0.5576450053690213,0.5574656479348947,best-head,exp/integrated/final/delivery-eta/baseline_plr-evaluation/2.toml,exp/integrated/final/delivery-eta/baseline_plr-best-head-evaluation/2,False,1638 +final,delivery-eta,baseline_plr,0,RMSE,rmse,lower,0.5553348954063525,-0.5553348954063525,0.550360417400049,greedy-heads,exp/integrated/final/delivery-eta/baseline_plr-evaluation/0.toml,exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/0,False,2184 +final,delivery-eta,baseline_plr,1,RMSE,rmse,lower,0.5553836483690917,-0.5553836483690917,0.5515298017800222,greedy-heads,exp/integrated/final/delivery-eta/baseline_plr-evaluation/1.toml,exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/1,False,1638 +final,delivery-eta,baseline_plr,2,RMSE,rmse,lower,0.5554437183690072,-0.5554437183690072,0.5516331354574737,greedy-heads,exp/integrated/final/delivery-eta/baseline_plr-evaluation/2.toml,exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/2,False,1638 +final,delivery-eta,best_rla_only,0,RMSE,rmse,lower,0.5572652691682626,-0.5572652691682626,0.5524207799310179,best-head,exp/integrated/final/delivery-eta/best_rla_only-evaluation/0.toml,exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/0,False,1365 +final,delivery-eta,best_rla_only,1,RMSE,rmse,lower,0.5577909128836495,-0.5577909128836495,0.5532065757037241,best-head,exp/integrated/final/delivery-eta/best_rla_only-evaluation/1.toml,exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/1,False,1365 +final,delivery-eta,best_rla_only,2,RMSE,rmse,lower,0.5585470113340857,-0.5585470113340857,0.5573737419749005,best-head,exp/integrated/final/delivery-eta/best_rla_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/2,False,1911 +final,delivery-eta,best_rla_only,0,RMSE,rmse,lower,0.5553930389270275,-0.5553930389270275,0.5519483119271499,greedy-heads,exp/integrated/final/delivery-eta/best_rla_only-evaluation/0.toml,exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/0,False,1365 +final,delivery-eta,best_rla_only,1,RMSE,rmse,lower,0.5557605143473513,-0.5557605143473513,0.5514412571081766,greedy-heads,exp/integrated/final/delivery-eta/best_rla_only-evaluation/1.toml,exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/1,False,1365 +final,delivery-eta,best_rla_only,2,RMSE,rmse,lower,0.5554844141815782,-0.5554844141815782,0.5520847398512095,greedy-heads,exp/integrated/final/delivery-eta/best_rla_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/2,False,1911 +final,delivery-eta,best_esam_only,2,RMSE,rmse,lower,0.5576450053690213,-0.5576450053690213,0.5574656479348947,best-head,exp/integrated/final/delivery-eta/best_esam_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_esam_only-best-head-evaluation/2,False,1638 +final,delivery-eta,best_esam_only,1,RMSE,rmse,lower,0.5578575885405295,-0.5578575885405295,0.5568376961917461,best-head,exp/integrated/final/delivery-eta/best_esam_only-evaluation/1.toml,exp/integrated/final/delivery-eta/best_esam_only-best-head-evaluation/1,False,1638 +final,delivery-eta,best_esam_only,0,RMSE,rmse,lower,0.5596782586864594,-0.5596782586864594,0.5565908577070521,best-head,exp/integrated/final/delivery-eta/best_esam_only-evaluation/0.toml,exp/integrated/final/delivery-eta/best_esam_only-best-head-evaluation/0,False,2730 +final,delivery-eta,best_esam_only,2,RMSE,rmse,lower,0.5554437183690072,-0.5554437183690072,0.5516331354574737,greedy-heads,exp/integrated/final/delivery-eta/best_esam_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/2,False,1638 +final,delivery-eta,best_esam_only,1,RMSE,rmse,lower,0.5553836483690917,-0.5553836483690917,0.5515298017800222,greedy-heads,exp/integrated/final/delivery-eta/best_esam_only-evaluation/1.toml,exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/1,False,1638 +final,delivery-eta,best_esam_only,0,RMSE,rmse,lower,0.555452544543325,-0.555452544543325,0.5532771704027001,greedy-heads,exp/integrated/final/delivery-eta/best_esam_only-evaluation/0.toml,exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/0,False,2730 +final,delivery-eta,best_mfb_only,1,RMSE,rmse,lower,0.5576769100347859,-0.5576769100347859,0.5558933529149844,best-head,exp/integrated/final/delivery-eta/best_mfb_only-evaluation/1.toml,exp/integrated/final/delivery-eta/best_mfb_only-best-head-evaluation/1,False,1638 +final,delivery-eta,best_mfb_only,2,RMSE,rmse,lower,0.558478390179117,-0.558478390179117,0.5526173066624255,best-head,exp/integrated/final/delivery-eta/best_mfb_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_mfb_only-best-head-evaluation/2,False,1638 +final,delivery-eta,best_mfb_only,0,RMSE,rmse,lower,0.5590051856389011,-0.5590051856389011,0.5534488118080276,best-head,exp/integrated/final/delivery-eta/best_mfb_only-evaluation/0.toml,exp/integrated/final/delivery-eta/best_mfb_only-best-head-evaluation/0,False,2184 +final,delivery-eta,best_mfb_only,1,RMSE,rmse,lower,0.55527543085353,-0.55527543085353,0.5524079939671243,greedy-heads,exp/integrated/final/delivery-eta/best_mfb_only-evaluation/1.toml,exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/1,False,1638 +final,delivery-eta,best_mfb_only,2,RMSE,rmse,lower,0.5553606274153348,-0.5553606274153348,0.5524157087402433,greedy-heads,exp/integrated/final/delivery-eta/best_mfb_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/2,False,1638 +final,delivery-eta,best_mfb_only,0,RMSE,rmse,lower,0.5553099940581016,-0.5553099940581016,0.5510631408064661,greedy-heads,exp/integrated/final/delivery-eta/best_mfb_only-evaluation/0.toml,exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/0,False,2184 +final,delivery-eta,best_cf_fisd_only,2,RMSE,rmse,lower,0.5576450053690213,-0.5576450053690213,0.5574656479348947,best-head,exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_cf_fisd_only-best-head-evaluation/2,False,1638 +final,delivery-eta,best_cf_fisd_only,1,RMSE,rmse,lower,0.5578575885405295,-0.5578575885405295,0.5568376961917461,best-head,exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/1.toml,exp/integrated/final/delivery-eta/best_cf_fisd_only-best-head-evaluation/1,False,1638 +final,delivery-eta,best_cf_fisd_only,0,RMSE,rmse,lower,0.5583881453279098,-0.5583881453279098,0.5557654209580059,best-head,exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/0.toml,exp/integrated/final/delivery-eta/best_cf_fisd_only-best-head-evaluation/0,False,2184 +final,delivery-eta,best_cf_fisd_only,2,RMSE,rmse,lower,0.5554437183690072,-0.5554437183690072,0.5516331354574737,greedy-heads,exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/2,False,1638 +final,delivery-eta,best_cf_fisd_only,1,RMSE,rmse,lower,0.5553836483690917,-0.5553836483690917,0.5515298017800222,greedy-heads,exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/1.toml,exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/1,False,1638 +final,delivery-eta,best_cf_fisd_only,0,RMSE,rmse,lower,0.5553348954063525,-0.5553348954063525,0.550360417400049,greedy-heads,exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/0.toml,exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/0,False,2184 +final,delivery-eta,best_combined,1,RMSE,rmse,lower,0.5577909128836495,-0.5577909128836495,0.5532065757037241,best-head,exp/integrated/final/delivery-eta/best_combined-evaluation/1.toml,exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/1,False,1365 +final,delivery-eta,best_combined,0,RMSE,rmse,lower,0.5572652691682626,-0.5572652691682626,0.5524207799310179,best-head,exp/integrated/final/delivery-eta/best_combined-evaluation/0.toml,exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/0,False,1365 +final,delivery-eta,best_combined,2,RMSE,rmse,lower,0.5585470113340857,-0.5585470113340857,0.5573737419749005,best-head,exp/integrated/final/delivery-eta/best_combined-evaluation/2.toml,exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/2,False,1911 +final,delivery-eta,best_combined,1,RMSE,rmse,lower,0.5557605143473513,-0.5557605143473513,0.5514412571081766,greedy-heads,exp/integrated/final/delivery-eta/best_combined-evaluation/1.toml,exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/1,False,1365 +final,delivery-eta,best_combined,0,RMSE,rmse,lower,0.5553930389270275,-0.5553930389270275,0.5519483119271499,greedy-heads,exp/integrated/final/delivery-eta/best_combined-evaluation/0.toml,exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/0,False,1365 +final,delivery-eta,best_combined,2,RMSE,rmse,lower,0.5554844141815782,-0.5554844141815782,0.5520847398512095,greedy-heads,exp/integrated/final/delivery-eta/best_combined-evaluation/2.toml,exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/2,False,1911 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n0_lr0p25_fr0,0,RMSE,rmse,lower,0.5558749906446844,-0.5558749906446844,0.5496162918403876,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n0_lr0p25_fr0p5,0,RMSE,rmse,lower,0.5558749906446844,-0.5558749906446844,0.5496162918403876,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0p5-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n0_lr0p5_fr0,0,RMSE,rmse,lower,0.5558749906446844,-0.5558749906446844,0.5496162918403876,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n0_lr0p5_fr0p5,0,RMSE,rmse,lower,0.5558749906446844,-0.5558749906446844,0.5496162918403876,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0p5-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n1em05_lr0p25_fr0,0,RMSE,rmse,lower,0.5558749906446844,-0.5558749906446844,0.5496162918403876,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n1em05_lr0p25_fr0p5,0,RMSE,rmse,lower,0.5558749906446844,-0.5558749906446844,0.5496162918403876,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0p5-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n1em05_lr0p5_fr0,0,RMSE,rmse,lower,0.5558749906446844,-0.5558749906446844,0.5496162918403876,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n1em05_lr0p5_fr0p5,0,RMSE,rmse,lower,0.5558749906446844,-0.5558749906446844,0.5496162918403876,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0p5-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n0p0001_lr0p25_fr0,0,RMSE,rmse,lower,0.5558749906446844,-0.5558749906446844,0.5496162918403876,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n0p0001_lr0p25_fr0p5,0,RMSE,rmse,lower,0.5558749906446844,-0.5558749906446844,0.5496162918403876,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0p5-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n0p0001_lr0p5_fr0,0,RMSE,rmse,lower,0.5558749906446844,-0.5558749906446844,0.5496162918403876,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n0p0001_lr0p5_fr0p5,0,RMSE,rmse,lower,0.5558749906446844,-0.5558749906446844,0.5496162918403876,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0p5-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n0_lr0p25_fr0,0,RMSE,rmse,lower,0.5558022325623276,-0.5558022325623276,0.5512480144508224,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n0_lr0p25_fr0p5,0,RMSE,rmse,lower,0.5558022325623276,-0.5558022325623276,0.5512480144508224,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0p5-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n0_lr0p5_fr0,0,RMSE,rmse,lower,0.5558360123700841,-0.5558360123700841,0.5491424149361828,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0-evaluation/0,False,3276 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n0_lr0p5_fr0p5,0,RMSE,rmse,lower,0.5558360123700841,-0.5558360123700841,0.5491424149361828,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0p5-evaluation/0,False,3276 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n1em05_lr0p25_fr0,0,RMSE,rmse,lower,0.5560690371152802,-0.5560690371152802,0.5483022050176959,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0-evaluation/0,False,3276 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n1em05_lr0p25_fr0p5,0,RMSE,rmse,lower,0.5558022325623276,-0.5558022325623276,0.5512480144508224,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0p5-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n1em05_lr0p5_fr0,0,RMSE,rmse,lower,0.5557362219646558,-0.5557362219646558,0.5505742164168008,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n1em05_lr0p5_fr0p5,0,RMSE,rmse,lower,0.5558360123700841,-0.5558360123700841,0.5491424149361828,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0p5-evaluation/0,False,3276 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n0p0001_lr0p25_fr0,0,RMSE,rmse,lower,0.5555827210327928,-0.5555827210327928,0.5515477953887775,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n0p0001_lr0p25_fr0p5,0,RMSE,rmse,lower,0.5558022325623276,-0.5558022325623276,0.5512480144508224,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0p5-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n0p0001_lr0p5_fr0,0,RMSE,rmse,lower,0.5557252820018314,-0.5557252820018314,0.5515971532207388,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n0p0001_lr0p5_fr0p5,0,RMSE,rmse,lower,0.5558360123700841,-0.5558360123700841,0.5491424149361828,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0p5-evaluation/0,False,3276 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n0_lr0p25_fr0,0,RMSE,rmse,lower,0.5558022325623276,-0.5558022325623276,0.5512480144508224,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n0_lr0p25_fr0p5,0,RMSE,rmse,lower,0.5558022325623276,-0.5558022325623276,0.5512480144508224,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0p5-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n0_lr0p5_fr0,0,RMSE,rmse,lower,0.5558360123700841,-0.5558360123700841,0.5491424149361828,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0-evaluation/0,False,3276 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n0_lr0p5_fr0p5,0,RMSE,rmse,lower,0.5558360123700841,-0.5558360123700841,0.5491424149361828,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0p5-evaluation/0,False,3276 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n1em05_lr0p25_fr0,0,RMSE,rmse,lower,0.5560380585478252,-0.5560380585478252,0.5494440227195571,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0-evaluation/0,False,3276 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n1em05_lr0p25_fr0p5,0,RMSE,rmse,lower,0.5558022325623276,-0.5558022325623276,0.5512480144508224,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0p5-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n1em05_lr0p5_fr0,0,RMSE,rmse,lower,0.5560025757843153,-0.5560025757843153,0.550425719096006,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0-evaluation/0,False,1365 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n1em05_lr0p5_fr0p5,0,RMSE,rmse,lower,0.5558360123700841,-0.5558360123700841,0.5491424149361828,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0p5-evaluation/0,False,3276 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n0p0001_lr0p25_fr0,0,RMSE,rmse,lower,0.5564917917545931,-0.5564917917545931,0.5481810097693468,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0-evaluation/0,False,3276 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n0p0001_lr0p25_fr0p5,0,RMSE,rmse,lower,0.5558022325623276,-0.5558022325623276,0.5512480144508224,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0p5-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n0p0001_lr0p5_fr0,0,RMSE,rmse,lower,0.5559356776481553,-0.5559356776481553,0.55135486033622,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0-evaluation/0,False,1365 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n0p0001_lr0p5_fr0p5,0,RMSE,rmse,lower,0.5558360123700841,-0.5558360123700841,0.5491424149361828,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0p5-evaluation/0,False,3276 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p00025_sf0p25_adaptert,0,RMSE,rmse,lower,0.5558750174513605,-0.5558750174513605,0.5496162918403876,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adaptert-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adaptert-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p00025_sf0p25_adapterf,0,RMSE,rmse,lower,0.5558749906446844,-0.5558749906446844,0.5496162918403876,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adapterf-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adapterf-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p00025_sf0p5_adaptert,0,RMSE,rmse,lower,0.5558750174513605,-0.5558750174513605,0.5496162918403876,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adaptert-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adaptert-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p00025_sf0p5_adapterf,0,RMSE,rmse,lower,0.5558749906446844,-0.5558749906446844,0.5496162918403876,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adapterf-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adapterf-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p0005_sf0p25_adaptert,0,RMSE,rmse,lower,0.5558750174513605,-0.5558750174513605,0.5496162918403876,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adaptert-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adaptert-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p0005_sf0p25_adapterf,0,RMSE,rmse,lower,0.5558749906446844,-0.5558749906446844,0.5496162918403876,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adapterf-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adapterf-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p0005_sf0p5_adaptert,0,RMSE,rmse,lower,0.5558750174513605,-0.5558750174513605,0.5496162918403876,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adaptert-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adaptert-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p0005_sf0p5_adapterf,0,RMSE,rmse,lower,0.5558749906446844,-0.5558749906446844,0.5496162918403876,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adapterf-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adapterf-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p001_sf0p25_adaptert,0,RMSE,rmse,lower,0.5558750174513605,-0.5558750174513605,0.5496162918403876,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adaptert-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adaptert-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p001_sf0p25_adapterf,0,RMSE,rmse,lower,0.5558269510040955,-0.5558269510040955,0.5527269612276788,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adapterf-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adapterf-evaluation/0,False,2730 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p001_sf0p5_adaptert,0,RMSE,rmse,lower,0.5558750174513605,-0.5558750174513605,0.5496162918403876,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adaptert-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adaptert-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p001_sf0p5_adapterf,0,RMSE,rmse,lower,0.5558749906446844,-0.5558749906446844,0.5496162918403876,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adapterf-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adapterf-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_mfb_k0p975_sf0p5_numerical_only,0,RMSE,rmse,lower,0.5558048331442785,-0.5558048331442785,0.5501619198178428,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_numerical_only-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_numerical_only-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_mfb_k0p975_sf0p5_per_member,0,RMSE,rmse,lower,0.5558048331442785,-0.5558048331442785,0.5501619198178428,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_per_member-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_per_member-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_mfb_k0p975_sf0p75_numerical_only,0,RMSE,rmse,lower,0.5558048331442785,-0.5558048331442785,0.5501619198178428,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_numerical_only-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_numerical_only-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_mfb_k0p975_sf0p75_per_member,0,RMSE,rmse,lower,0.5558048331442785,-0.5558048331442785,0.5501619198178428,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_per_member-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_per_member-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_mfb_k0p99_sf0p5_numerical_only,0,RMSE,rmse,lower,0.5558106240871751,-0.5558106240871751,0.5498568034160569,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_numerical_only-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_numerical_only-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_mfb_k0p99_sf0p5_per_member,0,RMSE,rmse,lower,0.5558106240871751,-0.5558106240871751,0.5498568034160569,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_per_member-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_per_member-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_mfb_k0p99_sf0p75_numerical_only,0,RMSE,rmse,lower,0.5558106240871751,-0.5558106240871751,0.5498568034160569,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_numerical_only-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_numerical_only-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_mfb_k0p99_sf0p75_per_member,0,RMSE,rmse,lower,0.5558106240871751,-0.5558106240871751,0.5498568034160569,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_per_member-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_per_member-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_cf_l0p005_consensus_raw,0,RMSE,rmse,lower,0.5558749906446844,-0.5558749906446844,0.5496162918403876,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_consensus_raw-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_consensus_raw-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_cf_l0p005_softmax,0,RMSE,rmse,lower,0.5558749906446844,-0.5558749906446844,0.5496162918403876,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_softmax-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_softmax-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_cf_l0p01_consensus_raw,0,RMSE,rmse,lower,0.5558749906446844,-0.5558749906446844,0.5496162918403876,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_consensus_raw-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_consensus_raw-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_cf_l0p01_softmax,0,RMSE,rmse,lower,0.5558749906446844,-0.5558749906446844,0.5496162918403876,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_softmax-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_softmax-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_cf_l0p02_consensus_raw,0,RMSE,rmse,lower,0.5558749906446844,-0.5558749906446844,0.5496162918403876,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_consensus_raw-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_consensus_raw-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_cf_l0p02_softmax,0,RMSE,rmse,lower,0.5558749906446844,-0.5558749906446844,0.5496162918403876,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_softmax-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_softmax-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n0_lr0p5_fr0,0,RMSE,rmse,lower,0.5583881453279098,-0.5583881453279098,0.5557654209580059,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n0p0001_lr0p5_fr0p5,0,RMSE,rmse,lower,0.5583881453279098,-0.5583881453279098,0.5557654209580059,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0p5-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n0_lr0p5_fr0,0,RMSE,rmse,lower,0.5553348954063525,-0.5553348954063525,0.550360417400049,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n0p0001_lr0p5_fr0p5,0,RMSE,rmse,lower,0.5553348954063525,-0.5553348954063525,0.550360417400049,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0p5-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n0p0001_lr0p25_fr0p5,0,RMSE,rmse,lower,0.5583881453279098,-0.5583881453279098,0.5557654209580059,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0p5-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n1em05_lr0p25_fr0,0,RMSE,rmse,lower,0.5583881453279098,-0.5583881453279098,0.5557654209580059,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n0p0001_lr0p25_fr0p5,0,RMSE,rmse,lower,0.5553348954063525,-0.5553348954063525,0.550360417400049,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0p5-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n1em05_lr0p25_fr0,0,RMSE,rmse,lower,0.5553348954063525,-0.5553348954063525,0.550360417400049,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n0p0001_lr0p25_fr0,0,RMSE,rmse,lower,0.5583881453279098,-0.5583881453279098,0.5557654209580059,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n0_lr0p25_fr0,0,RMSE,rmse,lower,0.5583881453279098,-0.5583881453279098,0.5557654209580059,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n0_lr0p5_fr0p5,0,RMSE,rmse,lower,0.5583881453279098,-0.5583881453279098,0.5557654209580059,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0p5-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n1em05_lr0p5_fr0,0,RMSE,rmse,lower,0.5583881453279098,-0.5583881453279098,0.5557654209580059,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n0p0001_lr0p25_fr0,0,RMSE,rmse,lower,0.5553348954063525,-0.5553348954063525,0.550360417400049,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n1em05_lr0p5_fr0p5,0,RMSE,rmse,lower,0.5583881453279098,-0.5583881453279098,0.5557654209580059,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0p5-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n0p0001_lr0p5_fr0,0,RMSE,rmse,lower,0.5583881453279098,-0.5583881453279098,0.5557654209580059,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n0_lr0p25_fr0,0,RMSE,rmse,lower,0.5553348954063525,-0.5553348954063525,0.550360417400049,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n0_lr0p5_fr0p5,0,RMSE,rmse,lower,0.5553348954063525,-0.5553348954063525,0.550360417400049,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0p5-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n1em05_lr0p5_fr0,0,RMSE,rmse,lower,0.5553348954063525,-0.5553348954063525,0.550360417400049,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n1em05_lr0p5_fr0p5,0,RMSE,rmse,lower,0.5553348954063525,-0.5553348954063525,0.550360417400049,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0p5-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n0p0001_lr0p5_fr0,0,RMSE,rmse,lower,0.5553348954063525,-0.5553348954063525,0.550360417400049,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n0_lr0p25_fr0p5,0,RMSE,rmse,lower,0.5583881453279098,-0.5583881453279098,0.5557654209580059,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0p5-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n1em05_lr0p25_fr0p5,0,RMSE,rmse,lower,0.5583881453279098,-0.5583881453279098,0.5557654209580059,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0p5-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n0_lr0p25_fr0p5,0,RMSE,rmse,lower,0.5553348954063525,-0.5553348954063525,0.550360417400049,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0p5-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n1em05_lr0p25_fr0p5,0,RMSE,rmse,lower,0.5553348954063525,-0.5553348954063525,0.550360417400049,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0p5-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n0_lr0p25_fr0,0,RMSE,rmse,lower,0.5593939165294649,-0.5593939165294649,0.5608508784321002,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n0_lr0p25_fr0,0,RMSE,rmse,lower,0.5554104781133956,-0.5554104781133956,0.5525622419967886,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n0_lr0p25_fr0p5,0,RMSE,rmse,lower,0.5593939165294649,-0.5593939165294649,0.5608508784321002,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0p5-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n0_lr0p25_fr0p5,0,RMSE,rmse,lower,0.5554104781133956,-0.5554104781133956,0.5525622419967886,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0p5-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n0_lr0p5_fr0p5,0,RMSE,rmse,lower,0.5596610856119687,-0.5596610856119687,0.5563749515907954,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0p5-best-head-evaluation/0,False,3276 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n0_lr0p5_fr0,0,RMSE,rmse,lower,0.5596610856119687,-0.5596610856119687,0.5563749515907954,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0-best-head-evaluation/0,False,3276 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n0_lr0p5_fr0p5,0,RMSE,rmse,lower,0.5552555453020565,-0.5552555453020565,0.551575730248866,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0p5-greedy-heads-evaluation/0,False,3276 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n0_lr0p5_fr0,0,RMSE,rmse,lower,0.5552555453020565,-0.5552555453020565,0.551575730248866,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0-greedy-heads-evaluation/0,False,3276 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n1em05_lr0p25_fr0p5,0,RMSE,rmse,lower,0.5593939165294649,-0.5593939165294649,0.5608508784321002,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0p5-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n1em05_lr0p25_fr0p5,0,RMSE,rmse,lower,0.5554104781133956,-0.5554104781133956,0.5525622419967886,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0p5-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n1em05_lr0p5_fr0,0,RMSE,rmse,lower,0.5583888658501774,-0.5583888658501774,0.5543243670215346,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n1em05_lr0p5_fr0,0,RMSE,rmse,lower,0.5551940861270195,-0.5551940861270195,0.5519268755870242,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n0p0001_lr0p25_fr0p5,0,RMSE,rmse,lower,0.5593939165294649,-0.5593939165294649,0.5608508784321002,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0p5-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n0p0001_lr0p25_fr0p5,0,RMSE,rmse,lower,0.5554104781133956,-0.5554104781133956,0.5525622419967886,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0p5-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n0p0001_lr0p25_fr0,0,RMSE,rmse,lower,0.5586283212914263,-0.5586283212914263,0.5508851568810451,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n0p0001_lr0p25_fr0,0,RMSE,rmse,lower,0.5550703688421238,-0.5550703688421238,0.5524078860674355,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n0p0001_lr0p5_fr0,0,RMSE,rmse,lower,0.5584356978034605,-0.5584356978034605,0.5561328378795123,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n0p0001_lr0p5_fr0,0,RMSE,rmse,lower,0.555272076390732,-0.555272076390732,0.5536843483543759,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n1em05_lr0p25_fr0,0,RMSE,rmse,lower,0.5603404179951016,-0.5603404179951016,0.5552489703004402,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0-best-head-evaluation/0,False,3276 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n1em05_lr0p25_fr0,0,RMSE,rmse,lower,0.5554431013363782,-0.5554431013363782,0.5505825252366326,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0-greedy-heads-evaluation/0,False,3276 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n1em05_lr0p5_fr0p5,0,RMSE,rmse,lower,0.5596610856119687,-0.5596610856119687,0.5563749515907954,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0p5-best-head-evaluation/0,False,3276 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n1em05_lr0p5_fr0p5,0,RMSE,rmse,lower,0.5552555453020565,-0.5552555453020565,0.551575730248866,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0p5-greedy-heads-evaluation/0,False,3276 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n0p0001_lr0p5_fr0p5,0,RMSE,rmse,lower,0.5596610856119687,-0.5596610856119687,0.5563749515907954,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0p5-best-head-evaluation/0,False,3276 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n0p0001_lr0p5_fr0p5,0,RMSE,rmse,lower,0.5552555453020565,-0.5552555453020565,0.551575730248866,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0p5-greedy-heads-evaluation/0,False,3276 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n0_lr0p25_fr0,0,RMSE,rmse,lower,0.5593939165294649,-0.5593939165294649,0.5608508784321002,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n0_lr0p25_fr0p5,0,RMSE,rmse,lower,0.5593939165294649,-0.5593939165294649,0.5608508784321002,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0p5-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n0_lr0p25_fr0,0,RMSE,rmse,lower,0.5554104781133956,-0.5554104781133956,0.5525622419967886,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n0_lr0p25_fr0p5,0,RMSE,rmse,lower,0.5554104781133956,-0.5554104781133956,0.5525622419967886,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0p5-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n0_lr0p5_fr0,0,RMSE,rmse,lower,0.5596610856119687,-0.5596610856119687,0.5563749515907954,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0-best-head-evaluation/0,False,3276 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n0_lr0p5_fr0,0,RMSE,rmse,lower,0.5552555453020565,-0.5552555453020565,0.551575730248866,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0-greedy-heads-evaluation/0,False,3276 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n0_lr0p5_fr0p5,0,RMSE,rmse,lower,0.5596610856119687,-0.5596610856119687,0.5563749515907954,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0p5-best-head-evaluation/0,False,3276 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n0_lr0p5_fr0p5,0,RMSE,rmse,lower,0.5552555453020565,-0.5552555453020565,0.551575730248866,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0p5-greedy-heads-evaluation/0,False,3276 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n1em05_lr0p5_fr0,0,RMSE,rmse,lower,0.5579574268466545,-0.5579574268466545,0.5527197091141781,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0-best-head-evaluation/0,False,1365 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n1em05_lr0p5_fr0,0,RMSE,rmse,lower,0.5553178831836687,-0.5553178831836687,0.5514062081885815,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0-greedy-heads-evaluation/0,False,1365 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n1em05_lr0p25_fr0p5,0,RMSE,rmse,lower,0.5593939165294649,-0.5593939165294649,0.5608508784321002,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0p5-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n1em05_lr0p25_fr0p5,0,RMSE,rmse,lower,0.5554104781133956,-0.5554104781133956,0.5525622419967886,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0p5-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p00025_sf0p5_adaptert,0,RMSE,rmse,lower,0.5583881453279098,-0.5583881453279098,0.5557654209580059,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adaptert-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adaptert-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p00025_sf0p5_adaptert,0,RMSE,rmse,lower,0.5553348954063525,-0.5553348954063525,0.550360417400049,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adaptert-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adaptert-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p00025_sf0p25_adaptert,0,RMSE,rmse,lower,0.5583881453279098,-0.5583881453279098,0.5557654209580059,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adaptert-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adaptert-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p00025_sf0p25_adaptert,0,RMSE,rmse,lower,0.5553348954063525,-0.5553348954063525,0.550360417400049,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adaptert-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adaptert-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p00025_sf0p5_adapterf,0,RMSE,rmse,lower,0.5583881453279098,-0.5583881453279098,0.5557654209580059,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adapterf-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adapterf-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p00025_sf0p5_adapterf,0,RMSE,rmse,lower,0.5553348954063525,-0.5553348954063525,0.550360417400049,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adapterf-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adapterf-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n1em05_lr0p25_fr0,0,RMSE,rmse,lower,0.5601935253217213,-0.5601935253217213,0.5598375837203123,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0-best-head-evaluation/0,False,3276 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n1em05_lr0p25_fr0,0,RMSE,rmse,lower,0.5553281335127394,-0.5553281335127394,0.550823697016921,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0-greedy-heads-evaluation/0,False,3276 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p00025_sf0p25_adapterf,0,RMSE,rmse,lower,0.5583881453279098,-0.5583881453279098,0.5557654209580059,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adapterf-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adapterf-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p00025_sf0p25_adapterf,0,RMSE,rmse,lower,0.5553348954063525,-0.5553348954063525,0.550360417400049,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adapterf-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adapterf-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n1em05_lr0p5_fr0p5,0,RMSE,rmse,lower,0.5596610856119687,-0.5596610856119687,0.5563749515907954,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0p5-best-head-evaluation/0,False,3276 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n1em05_lr0p5_fr0p5,0,RMSE,rmse,lower,0.5552555453020565,-0.5552555453020565,0.551575730248866,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0p5-greedy-heads-evaluation/0,False,3276 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n0p0001_lr0p5_fr0,0,RMSE,rmse,lower,0.5572652691682626,-0.5572652691682626,0.5524207799310179,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0-best-head-evaluation/0,False,1365 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n0p0001_lr0p5_fr0,0,RMSE,rmse,lower,0.5553930389270275,-0.5553930389270275,0.5519483119271499,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0-greedy-heads-evaluation/0,False,1365 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n0p0001_lr0p25_fr0p5,0,RMSE,rmse,lower,0.5593939165294649,-0.5593939165294649,0.5608508784321002,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0p5-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n0p0001_lr0p25_fr0p5,0,RMSE,rmse,lower,0.5554104781133956,-0.5554104781133956,0.5525622419967886,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0p5-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p0005_sf0p25_adaptert,0,RMSE,rmse,lower,0.5583881453279098,-0.5583881453279098,0.5557654209580059,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adaptert-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adaptert-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p0005_sf0p25_adaptert,0,RMSE,rmse,lower,0.5553348954063525,-0.5553348954063525,0.550360417400049,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adaptert-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adaptert-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p0005_sf0p5_adaptert,0,RMSE,rmse,lower,0.5583881453279098,-0.5583881453279098,0.5557654209580059,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adaptert-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adaptert-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p0005_sf0p5_adaptert,0,RMSE,rmse,lower,0.5553348954063525,-0.5553348954063525,0.550360417400049,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adaptert-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adaptert-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p0005_sf0p25_adapterf,0,RMSE,rmse,lower,0.5583881453279098,-0.5583881453279098,0.5557654209580059,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adapterf-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adapterf-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p0005_sf0p25_adapterf,0,RMSE,rmse,lower,0.5553348954063525,-0.5553348954063525,0.550360417400049,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adapterf-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adapterf-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p0005_sf0p5_adapterf,0,RMSE,rmse,lower,0.5583881453279098,-0.5583881453279098,0.5557654209580059,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adapterf-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adapterf-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p0005_sf0p5_adapterf,0,RMSE,rmse,lower,0.5553348954063525,-0.5553348954063525,0.550360417400049,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adapterf-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adapterf-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_mfb_k0p975_sf0p5_numerical_only,0,RMSE,rmse,lower,0.5590051856389011,-0.5590051856389011,0.5534488118080276,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_numerical_only-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_numerical_only-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_mfb_k0p975_sf0p5_numerical_only,0,RMSE,rmse,lower,0.5553099940581016,-0.5553099940581016,0.5510631408064661,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_numerical_only-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_numerical_only-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n0p0001_lr0p25_fr0,0,RMSE,rmse,lower,0.56004920160056,-0.56004920160056,0.5537449525211285,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0-best-head-evaluation/0,False,3276 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n0p0001_lr0p25_fr0,0,RMSE,rmse,lower,0.5553600907849848,-0.5553600907849848,0.5504577714884789,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0-greedy-heads-evaluation/0,False,3276 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n0p0001_lr0p5_fr0p5,0,RMSE,rmse,lower,0.5596610856119687,-0.5596610856119687,0.5563749515907954,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0p5-best-head-evaluation/0,False,3276 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n0p0001_lr0p5_fr0p5,0,RMSE,rmse,lower,0.5552555453020565,-0.5552555453020565,0.551575730248866,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0p5-greedy-heads-evaluation/0,False,3276 +rescue_delivery_individual,delivery-eta,deliv_mfb_k0p975_sf0p5_per_member,0,RMSE,rmse,lower,0.5590051856389011,-0.5590051856389011,0.5534488118080276,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_per_member-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_per_member-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_mfb_k0p975_sf0p5_per_member,0,RMSE,rmse,lower,0.5553099940581016,-0.5553099940581016,0.5510631408064661,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_per_member-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_per_member-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p001_sf0p5_adaptert,0,RMSE,rmse,lower,0.5583881453279098,-0.5583881453279098,0.5557654209580059,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adaptert-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adaptert-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p001_sf0p5_adaptert,0,RMSE,rmse,lower,0.5553348954063525,-0.5553348954063525,0.550360417400049,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adaptert-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adaptert-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p001_sf0p25_adaptert,0,RMSE,rmse,lower,0.5583881453279098,-0.5583881453279098,0.5557654209580059,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adaptert-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adaptert-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p001_sf0p25_adaptert,0,RMSE,rmse,lower,0.5553348954063525,-0.5553348954063525,0.550360417400049,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adaptert-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adaptert-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p001_sf0p5_adapterf,0,RMSE,rmse,lower,0.5583881453279098,-0.5583881453279098,0.5557654209580059,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adapterf-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adapterf-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p001_sf0p5_adapterf,0,RMSE,rmse,lower,0.5553348954063525,-0.5553348954063525,0.550360417400049,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adapterf-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adapterf-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_mfb_k0p975_sf0p75_numerical_only,0,RMSE,rmse,lower,0.5590051856389011,-0.5590051856389011,0.5534488118080276,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_numerical_only-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_numerical_only-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_mfb_k0p975_sf0p75_numerical_only,0,RMSE,rmse,lower,0.5553099940581016,-0.5553099940581016,0.5510631408064661,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_numerical_only-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_numerical_only-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_mfb_k0p975_sf0p75_per_member,0,RMSE,rmse,lower,0.5590051856389011,-0.5590051856389011,0.5534488118080276,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_per_member-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_per_member-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_mfb_k0p975_sf0p75_per_member,0,RMSE,rmse,lower,0.5553099940581016,-0.5553099940581016,0.5510631408064661,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_per_member-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_per_member-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p001_sf0p25_adapterf,0,RMSE,rmse,lower,0.5596782586864594,-0.5596782586864594,0.5565908577070521,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adapterf-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adapterf-best-head-evaluation/0,False,2730 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p001_sf0p25_adapterf,0,RMSE,rmse,lower,0.555452544543325,-0.555452544543325,0.5532771704027001,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adapterf-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adapterf-greedy-heads-evaluation/0,False,2730 +rescue_delivery_individual,delivery-eta,deliv_mfb_k0p99_sf0p5_numerical_only,0,RMSE,rmse,lower,0.5584120821811833,-0.5584120821811833,0.5559629363941286,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_numerical_only-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_numerical_only-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_mfb_k0p99_sf0p5_numerical_only,0,RMSE,rmse,lower,0.5553168366734594,-0.5553168366734594,0.5509050649441617,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_numerical_only-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_numerical_only-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_mfb_k0p99_sf0p75_numerical_only,0,RMSE,rmse,lower,0.5584120821811833,-0.5584120821811833,0.5559629363941286,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_numerical_only-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_numerical_only-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_mfb_k0p99_sf0p75_numerical_only,0,RMSE,rmse,lower,0.5553168366734594,-0.5553168366734594,0.5509050649441617,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_numerical_only-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_numerical_only-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_mfb_k0p99_sf0p5_per_member,0,RMSE,rmse,lower,0.5584120821811833,-0.5584120821811833,0.5559629363941286,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_per_member-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_per_member-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_mfb_k0p99_sf0p5_per_member,0,RMSE,rmse,lower,0.5553168366734594,-0.5553168366734594,0.5509050649441617,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_per_member-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_per_member-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_mfb_k0p99_sf0p75_per_member,0,RMSE,rmse,lower,0.5584120821811833,-0.5584120821811833,0.5559629363941286,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_per_member-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_per_member-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_mfb_k0p99_sf0p75_per_member,0,RMSE,rmse,lower,0.5553168366734594,-0.5553168366734594,0.5509050649441617,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_per_member-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_per_member-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_cf_l0p005_consensus_raw,0,RMSE,rmse,lower,0.5583881453279098,-0.5583881453279098,0.5557654209580059,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_consensus_raw-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_consensus_raw-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_cf_l0p005_consensus_raw,0,RMSE,rmse,lower,0.5553348954063525,-0.5553348954063525,0.550360417400049,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_consensus_raw-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_consensus_raw-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_cf_l0p005_softmax,0,RMSE,rmse,lower,0.5583881453279098,-0.5583881453279098,0.5557654209580059,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_softmax-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_softmax-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_cf_l0p005_softmax,0,RMSE,rmse,lower,0.5553348954063525,-0.5553348954063525,0.550360417400049,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_softmax-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_softmax-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_cf_l0p01_softmax,0,RMSE,rmse,lower,0.5583881453279098,-0.5583881453279098,0.5557654209580059,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_softmax-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_softmax-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_cf_l0p01_softmax,0,RMSE,rmse,lower,0.5553348954063525,-0.5553348954063525,0.550360417400049,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_softmax-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_softmax-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_cf_l0p01_consensus_raw,0,RMSE,rmse,lower,0.5583881453279098,-0.5583881453279098,0.5557654209580059,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_consensus_raw-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_consensus_raw-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_cf_l0p01_consensus_raw,0,RMSE,rmse,lower,0.5553348954063525,-0.5553348954063525,0.550360417400049,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_consensus_raw-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_consensus_raw-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_cf_l0p02_consensus_raw,0,RMSE,rmse,lower,0.5583881453279098,-0.5583881453279098,0.5557654209580059,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_consensus_raw-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_consensus_raw-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_cf_l0p02_consensus_raw,0,RMSE,rmse,lower,0.5553348954063525,-0.5553348954063525,0.550360417400049,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_consensus_raw-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_consensus_raw-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_cf_l0p02_softmax,0,RMSE,rmse,lower,0.5583881453279098,-0.5583881453279098,0.5557654209580059,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_softmax-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_softmax-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_cf_l0p02_softmax,0,RMSE,rmse,lower,0.5553348954063525,-0.5553348954063525,0.550360417400049,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_softmax-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_softmax-greedy-heads-evaluation/0,False,2184 +rescue_delivery_combined,delivery-eta,deliv_comb_rla_esam,0,RMSE,rmse,lower,0.5555664137600386,-0.5555664137600386,0.5512279565776417,mean,exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam-evaluation/0.toml,exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam-evaluation/0,False,2730 +rescue_delivery_combined,delivery-eta,deliv_comb_rla_cf,0,RMSE,rmse,lower,0.5559356776481553,-0.5559356776481553,0.55135486033622,mean,exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_cf-evaluation/0.toml,exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_cf-evaluation/0,False,1365 +rescue_delivery_combined,delivery-eta,deliv_comb_esam_cf,0,RMSE,rmse,lower,0.5558269510040955,-0.5558269510040955,0.5527269612276788,mean,exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_esam_cf-evaluation/0.toml,exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_esam_cf-evaluation/0,False,2730 +rescue_delivery_combined,delivery-eta,deliv_comb_rla_esam_cf,0,RMSE,rmse,lower,0.5555664137600386,-0.5555664137600386,0.5512279565776417,mean,exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam_cf-evaluation/0.toml,exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam_cf-evaluation/0,False,2730 +rescue_delivery_combined,delivery-eta,deliv_comb_esam_cf,0,RMSE,rmse,lower,0.5596782586864594,-0.5596782586864594,0.5565908577070521,best-head,exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_esam_cf-evaluation/0.toml,exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_esam_cf-best-head-evaluation/0,False,2730 +rescue_delivery_combined,delivery-eta,deliv_comb_esam_cf,0,RMSE,rmse,lower,0.555452544543325,-0.555452544543325,0.5532771704027001,greedy-heads,exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_esam_cf-evaluation/0.toml,exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_esam_cf-greedy-heads-evaluation/0,False,2730 +rescue_delivery_combined,delivery-eta,deliv_comb_rla_cf,0,RMSE,rmse,lower,0.5572652691682626,-0.5572652691682626,0.5524207799310179,best-head,exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_cf-evaluation/0.toml,exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_cf-best-head-evaluation/0,False,1365 +rescue_delivery_combined,delivery-eta,deliv_comb_rla_cf,0,RMSE,rmse,lower,0.5553930389270275,-0.5553930389270275,0.5519483119271499,greedy-heads,exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_cf-evaluation/0.toml,exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_cf-greedy-heads-evaluation/0,False,1365 +rescue_delivery_combined,delivery-eta,deliv_comb_rla_esam,0,RMSE,rmse,lower,0.5591176650506209,-0.5591176650506209,0.5583917212440949,best-head,exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam-evaluation/0.toml,exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam-best-head-evaluation/0,False,2730 +rescue_delivery_combined,delivery-eta,deliv_comb_rla_esam,0,RMSE,rmse,lower,0.5550917643214267,-0.5550917643214267,0.5526470209655567,greedy-heads,exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam-evaluation/0.toml,exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam-greedy-heads-evaluation/0,False,2730 +rescue_delivery_combined,delivery-eta,deliv_comb_rla_esam_cf,0,RMSE,rmse,lower,0.5591176650506209,-0.5591176650506209,0.5583917212440949,best-head,exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam_cf-evaluation/0.toml,exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam_cf-best-head-evaluation/0,False,2730 +rescue_delivery_combined,delivery-eta,deliv_comb_rla_esam_cf,0,RMSE,rmse,lower,0.5550917643214267,-0.5550917643214267,0.5526470209655567,greedy-heads,exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam_cf-evaluation/0.toml,exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam_cf-greedy-heads-evaluation/0,False,2730 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0_l0p001,0,RMSE,rmse,lower,0.23516710262665957,-0.23516710262665957,0.2392893867678542,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p001-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0_l0p005,0,RMSE,rmse,lower,0.23541557264636437,-0.23541557264636437,0.23955821427368978,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p005-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0_l0p01,0,RMSE,rmse,lower,0.23516078988949235,-0.23516078988949235,0.24165682950771084,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p01-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0_l0p02,0,RMSE,rmse,lower,0.23537261362228468,-0.23537261362228468,0.23578993148696667,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p02-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n1em05_l0p001,0,RMSE,rmse,lower,0.23516710262665957,-0.23516710262665957,0.2392893867678542,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p001-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n1em05_l0p005,0,RMSE,rmse,lower,0.23541557264636437,-0.23541557264636437,0.23955821427368978,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p005-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n1em05_l0p01,0,RMSE,rmse,lower,0.23516078988949235,-0.23516078988949235,0.24165682950771084,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p01-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n1em05_l0p02,0,RMSE,rmse,lower,0.23537261362228468,-0.23537261362228468,0.23578993148696667,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p02-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0p0001_l0p001,0,RMSE,rmse,lower,0.23516710262665957,-0.23516710262665957,0.2392893867678542,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p001-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0p0001_l0p005,0,RMSE,rmse,lower,0.23541557264636437,-0.23541557264636437,0.23955821427368978,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p005-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0p0001_l0p01,0,RMSE,rmse,lower,0.23516078988949235,-0.23516078988949235,0.24165682950771084,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p01-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0p0001_l0p02,0,RMSE,rmse,lower,0.23537261362228468,-0.23537261362228468,0.23578993148696667,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p02-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0p001_l0p001,0,RMSE,rmse,lower,0.23516710262665957,-0.23516710262665957,0.2392893867678542,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p001-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0p001_l0p005,0,RMSE,rmse,lower,0.23541557264636437,-0.23541557264636437,0.23955821427368978,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p005-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0p001_l0p01,0,RMSE,rmse,lower,0.23516078988949235,-0.23516078988949235,0.24165682950771084,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p01-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0p001_l0p02,0,RMSE,rmse,lower,0.23537261362228468,-0.23537261362228468,0.23578993148696667,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p02-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0_l0p001,0,RMSE,rmse,lower,0.23516964509902982,-0.23516964509902982,0.23928895864366434,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p001-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0_l0p005,0,RMSE,rmse,lower,0.23541076200529618,-0.23541076200529618,0.2395584319829453,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p005-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0_l0p01,0,RMSE,rmse,lower,0.23516104335271928,-0.23516104335271928,0.24165658285763553,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p01-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0_l0p02,0,RMSE,rmse,lower,0.23537506682607195,-0.23537506682607195,0.23579029486809283,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p02-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n1em05_l0p001,0,RMSE,rmse,lower,0.23523322965051088,-0.23523322965051088,0.23438947156097584,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p001-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n1em05_l0p005,0,RMSE,rmse,lower,0.23519216180885402,-0.23519216180885402,0.23499174827687583,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p005-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n1em05_l0p01,0,RMSE,rmse,lower,0.23534793771825302,-0.23534793771825302,0.2358765820307155,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p01-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n1em05_l0p02,0,RMSE,rmse,lower,0.23499558463666298,-0.23499558463666298,0.23934769790965568,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p02-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0p0001_l0p001,0,RMSE,rmse,lower,0.23492518063329523,-0.23492518063329523,0.23863885185928208,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p001-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0p0001_l0p005,0,RMSE,rmse,lower,0.23501775344788706,-0.23501775344788706,0.23276117546354494,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p005-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0p0001_l0p01,0,RMSE,rmse,lower,0.23514059116024372,-0.23514059116024372,0.23749547910151964,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p01-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0p0001_l0p02,0,RMSE,rmse,lower,0.23506069808417399,-0.23506069808417399,0.24070191052163564,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p02-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0p001_l0p001,0,RMSE,rmse,lower,0.23558290278789318,-0.23558290278789318,0.24195079618816864,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p001-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0p001_l0p005,0,RMSE,rmse,lower,0.23531810633502007,-0.23531810633502007,0.2327283473545216,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p005-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0p001_l0p01,0,RMSE,rmse,lower,0.23480153213840635,-0.23480153213840635,0.23483096909658374,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p01-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0p001_l0p02,0,RMSE,rmse,lower,0.23510019641837185,-0.23510019641837185,0.23935828921252758,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p02-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0_l0p001,0,RMSE,rmse,lower,0.23516964509902982,-0.23516964509902982,0.23928895864366434,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p001-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0_l0p005,0,RMSE,rmse,lower,0.23541076200529618,-0.23541076200529618,0.2395584319829453,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p005-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0_l0p01,0,RMSE,rmse,lower,0.23516104335271928,-0.23516104335271928,0.24165658285763553,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p01-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0_l0p02,0,RMSE,rmse,lower,0.23537506682607195,-0.23537506682607195,0.23579029486809283,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p02-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n1em05_l0p001,0,RMSE,rmse,lower,0.23532450983472006,-0.23532450983472006,0.2376596833156001,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p001-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n1em05_l0p005,0,RMSE,rmse,lower,0.23472067454979642,-0.23472067454979642,0.23644700488851306,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p005-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n1em05_l0p01,0,RMSE,rmse,lower,0.23505473908329258,-0.23505473908329258,0.23281250512039894,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p01-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n1em05_l0p02,0,RMSE,rmse,lower,0.23511142271763394,-0.23511142271763394,0.23473994135203347,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p02-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0p0001_l0p001,0,RMSE,rmse,lower,0.2350763793606662,-0.2350763793606662,0.23832022238784234,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p001-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0p0001_l0p005,0,RMSE,rmse,lower,0.2350520527289723,-0.2350520527289723,0.23495368248701565,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p005-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0p0001_l0p01,0,RMSE,rmse,lower,0.23466412671675002,-0.23466412671675002,0.2324620941369229,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p01-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0p0001_l0p02,0,RMSE,rmse,lower,0.23532386870106714,-0.23532386870106714,0.23728542485175996,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p02-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0p001_l0p001,0,RMSE,rmse,lower,0.2349816418639806,-0.2349816418639806,0.24136844697035015,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p001-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0p001_l0p005,0,RMSE,rmse,lower,0.23549047304076126,-0.23549047304076126,0.24128326690664834,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p005-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0p001_l0p01,0,RMSE,rmse,lower,0.23543217967973734,-0.23543217967973734,0.23786624351296862,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p01-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0p001_l0p02,0,RMSE,rmse,lower,0.23514095554484465,-0.23514095554484465,0.23359667766611852,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p02-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0_rho0p00025,0,RMSE,rmse,lower,0.23484986994593354,-0.23484986994593354,0.23462197483665095,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p00025-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0_rho0p0005,0,RMSE,rmse,lower,0.23510992538054096,-0.23510992538054096,0.24175341990615093,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0005-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0_rho0p001,0,RMSE,rmse,lower,0.23467774706155475,-0.23467774706155475,0.23716614057842658,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p001-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0_rho0p0025,0,RMSE,rmse,lower,0.23474623366768377,-0.23474623366768377,0.23936563514587833,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0025-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n1em05_rho0p00025,0,RMSE,rmse,lower,0.23484986994593354,-0.23484986994593354,0.23462197483665095,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p00025-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n1em05_rho0p0005,0,RMSE,rmse,lower,0.23510992538054096,-0.23510992538054096,0.24175341990615093,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0005-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n1em05_rho0p001,0,RMSE,rmse,lower,0.23467774706155475,-0.23467774706155475,0.23716614057842658,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p001-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n1em05_rho0p0025,0,RMSE,rmse,lower,0.23474623366768377,-0.23474623366768377,0.23936563514587833,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0025-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0p0001_rho0p00025,0,RMSE,rmse,lower,0.23484986994593354,-0.23484986994593354,0.23462197483665095,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p00025-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0p0001_rho0p0005,0,RMSE,rmse,lower,0.23510992538054096,-0.23510992538054096,0.24175341990615093,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0005-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0p0001_rho0p001,0,RMSE,rmse,lower,0.23467774706155475,-0.23467774706155475,0.23716614057842658,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p001-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0p0001_rho0p0025,0,RMSE,rmse,lower,0.23474623366768377,-0.23474623366768377,0.23936563514587833,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0025-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0p001_rho0p00025,0,RMSE,rmse,lower,0.23484986994593354,-0.23484986994593354,0.23462197483665095,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p00025-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0p001_rho0p0005,0,RMSE,rmse,lower,0.23510992538054096,-0.23510992538054096,0.24175341990615093,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0005-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0p001_rho0p001,0,RMSE,rmse,lower,0.23467774706155475,-0.23467774706155475,0.23716614057842658,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p001-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0p001_rho0p0025,0,RMSE,rmse,lower,0.23474623366768377,-0.23474623366768377,0.23936563514587833,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0025-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0_rho0p00025,0,RMSE,rmse,lower,0.23499185132041592,-0.23499185132041592,0.2394094259267782,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p00025-evaluation/0,False,962 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0_rho0p0005,0,RMSE,rmse,lower,0.2346688971022881,-0.2346688971022881,0.23375485551605374,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0005-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0_rho0p001,0,RMSE,rmse,lower,0.23475735787864294,-0.23475735787864294,0.23961932837948463,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p001-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0_rho0p0025,0,RMSE,rmse,lower,0.23456885744734607,-0.23456885744734607,0.23654787235061092,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0025-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n1em05_rho0p00025,0,RMSE,rmse,lower,0.23516112255992166,-0.23516112255992166,0.23211789488202353,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p00025-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n1em05_rho0p0005,0,RMSE,rmse,lower,0.23464263099787872,-0.23464263099787872,0.23482863711969398,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0005-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n1em05_rho0p001,0,RMSE,rmse,lower,0.23461772747586743,-0.23461772747586743,0.23292056846137055,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p001-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n1em05_rho0p0025,0,RMSE,rmse,lower,0.23469026342966967,-0.23469026342966967,0.23333669987162928,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0025-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0p0001_rho0p00025,0,RMSE,rmse,lower,0.23496530422551093,-0.23496530422551093,0.23600846732737993,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p00025-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0p0001_rho0p0005,0,RMSE,rmse,lower,0.23480699780789907,-0.23480699780789907,0.23801010369895165,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0005-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0p0001_rho0p001,0,RMSE,rmse,lower,0.2356879335438401,-0.2356879335438401,0.23192085632684412,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p001-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0p0001_rho0p0025,0,RMSE,rmse,lower,0.2352761982909868,-0.2352761982909868,0.2333683170052332,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0025-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0p001_rho0p00025,0,RMSE,rmse,lower,0.23478980709452008,-0.23478980709452008,0.2332225120575295,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p00025-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0p001_rho0p0005,0,RMSE,rmse,lower,0.23527326904017234,-0.23527326904017234,0.24074289728147005,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0005-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0p001_rho0p001,0,RMSE,rmse,lower,0.23507535721736414,-0.23507535721736414,0.2433693260742304,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p001-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0p001_rho0p0025,0,RMSE,rmse,lower,0.23446964895916758,-0.23446964895916758,0.23505021426230255,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0025-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0_rho0p00025,0,RMSE,rmse,lower,0.23541561220714985,-0.23541561220714985,0.23578183426225294,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p00025-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0_rho0p0005,0,RMSE,rmse,lower,0.23501861733133902,-0.23501861733133902,0.23373303710107815,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0005-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0_rho0p001,0,RMSE,rmse,lower,0.23496360777291245,-0.23496360777291245,0.23914605469445158,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p001-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0_rho0p0025,0,RMSE,rmse,lower,0.23463987641889242,-0.23463987641889242,0.23470931052134267,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0025-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n1em05_rho0p00025,0,RMSE,rmse,lower,0.23507776598207258,-0.23507776598207258,0.2438242655494171,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p00025-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n1em05_rho0p0005,0,RMSE,rmse,lower,0.23477109182279635,-0.23477109182279635,0.2340687021877646,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0005-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n1em05_rho0p001,0,RMSE,rmse,lower,0.23488388446999084,-0.23488388446999084,0.23845471649958624,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p001-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n1em05_rho0p0025,0,RMSE,rmse,lower,0.23480782280356366,-0.23480782280356366,0.23834319169461882,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0025-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0p0001_rho0p00025,0,RMSE,rmse,lower,0.23513454704634784,-0.23513454704634784,0.2392125690267148,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p00025-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0p0001_rho0p0005,0,RMSE,rmse,lower,0.2354849520464804,-0.2354849520464804,0.23141729611827572,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0005-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0p0001_rho0p001,0,RMSE,rmse,lower,0.23554663270650014,-0.23554663270650014,0.24084128499653581,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p001-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0p0001_rho0p0025,0,RMSE,rmse,lower,0.23546283513255054,-0.23546283513255054,0.23448563984262677,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0025-evaluation/0,False,740 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0p001_rho0p00025,0,RMSE,rmse,lower,0.234735553294385,-0.234735553294385,0.23656269916289197,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p00025-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0p001_rho0p0005,0,RMSE,rmse,lower,0.2353183042208091,-0.2353183042208091,0.2326811298212561,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0005-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0p001_rho0p001,0,RMSE,rmse,lower,0.23492211221645545,-0.23492211221645545,0.23523946918101388,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p001-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0p001_rho0p0025,0,RMSE,rmse,lower,0.23496063498765302,-0.23496063498765302,0.2373818009282627,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0025-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p001,0,RMSE,rmse,lower,0.2357328737026148,-0.2357328737026148,0.2351772485906963,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p001-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p005,0,RMSE,rmse,lower,0.2351519581095747,-0.2351519581095747,0.2408751648791966,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p005-evaluation/0,False,1184 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p01,0,RMSE,rmse,lower,0.2353263382433216,-0.2353263382433216,0.2351845825591179,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p01-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p001,0,RMSE,rmse,lower,0.2347666170730829,-0.2347666170730829,0.23399684917333807,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p001-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p005,0,RMSE,rmse,lower,0.2351252944190024,-0.2351252944190024,0.23697534320527558,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p005-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p01,0,RMSE,rmse,lower,0.23512438339576403,-0.23512438339576403,0.2316782780837399,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p01-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001,0,RMSE,rmse,lower,0.235050071622046,-0.235050071622046,0.23184680318718037,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p005,0,RMSE,rmse,lower,0.23505585640746507,-0.23505585640746507,0.23276235981389973,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p005-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p01,0,RMSE,rmse,lower,0.23504784483792887,-0.23504784483792887,0.23465037063607563,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p01-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p001,0,RMSE,rmse,lower,0.2357328737026148,-0.2357328737026148,0.2351772485906963,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p001-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p005,0,RMSE,rmse,lower,0.2351519581095747,-0.2351519581095747,0.2408751648791966,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p005-evaluation/0,False,1184 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p01,0,RMSE,rmse,lower,0.2353263382433216,-0.2353263382433216,0.2351845825591179,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p01-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p001,0,RMSE,rmse,lower,0.2347666170730829,-0.2347666170730829,0.23399684917333807,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p001-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p005,0,RMSE,rmse,lower,0.2351252944190024,-0.2351252944190024,0.23697534320527558,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p005-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p01,0,RMSE,rmse,lower,0.23512438339576403,-0.23512438339576403,0.2316782780837399,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p01-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001,0,RMSE,rmse,lower,0.235050071622046,-0.235050071622046,0.23184680318718037,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p005,0,RMSE,rmse,lower,0.23505585640746507,-0.23505585640746507,0.23276235981389973,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p005-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p01,0,RMSE,rmse,lower,0.23504784483792887,-0.23504784483792887,0.23465037063607563,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p01-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p001,0,RMSE,rmse,lower,0.2357328737026148,-0.2357328737026148,0.2351772485906963,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p001-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p005,0,RMSE,rmse,lower,0.2351519581095747,-0.2351519581095747,0.2408751648791966,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p005-evaluation/0,False,1184 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p01,0,RMSE,rmse,lower,0.2353263382433216,-0.2353263382433216,0.2351845825591179,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p01-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p001,0,RMSE,rmse,lower,0.2347666170730829,-0.2347666170730829,0.23399684917333807,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p001-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p005,0,RMSE,rmse,lower,0.2351252944190024,-0.2351252944190024,0.23697534320527558,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p005-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p01,0,RMSE,rmse,lower,0.23512438339576403,-0.23512438339576403,0.2316782780837399,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p01-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p001,0,RMSE,rmse,lower,0.235050071622046,-0.235050071622046,0.23184680318718037,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p001-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p005,0,RMSE,rmse,lower,0.23505585640746507,-0.23505585640746507,0.23276235981389973,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p005-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p01,0,RMSE,rmse,lower,0.23504784483792887,-0.23504784483792887,0.23465037063607563,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p01-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p001,0,RMSE,rmse,lower,0.23493513883666117,-0.23493513883666117,0.2324465730511325,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p001-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p005,0,RMSE,rmse,lower,0.2350836134726586,-0.2350836134726586,0.23409832677668932,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p005-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p01,0,RMSE,rmse,lower,0.23493006464565203,-0.23493006464565203,0.23367776476223437,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p01-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p001,0,RMSE,rmse,lower,0.23512809084220013,-0.23512809084220013,0.2355998696072441,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p001-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p005,0,RMSE,rmse,lower,0.23518098688395506,-0.23518098688395506,0.23184570253369863,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p005-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p01,0,RMSE,rmse,lower,0.23473991754724038,-0.23473991754724038,0.23324559206850415,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p01-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p001,0,RMSE,rmse,lower,0.23468678716863678,-0.23468678716863678,0.23217868905016578,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p001-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p005,0,RMSE,rmse,lower,0.23531282668072034,-0.23531282668072034,0.2329136030473919,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p005-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p01,0,RMSE,rmse,lower,0.23489533520890316,-0.23489533520890316,0.23252367146541786,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p01-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p001,0,RMSE,rmse,lower,0.2349283758671495,-0.2349283758671495,0.23427005643589985,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p001-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p005,0,RMSE,rmse,lower,0.23481945176098018,-0.23481945176098018,0.23330914213736356,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p005-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p01,0,RMSE,rmse,lower,0.235230434477205,-0.235230434477205,0.23369840873687428,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p01-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p001,0,RMSE,rmse,lower,0.23518726739965692,-0.23518726739965692,0.23268638913916373,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p001-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p005,0,RMSE,rmse,lower,0.23483364211643792,-0.23483364211643792,0.23156642561711185,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p005-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p01,0,RMSE,rmse,lower,0.23457506700663386,-0.23457506700663386,0.2318295295556057,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p01-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p001,0,RMSE,rmse,lower,0.2351439260504182,-0.2351439260504182,0.23458613581335708,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p001-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p005,0,RMSE,rmse,lower,0.2346577290091983,-0.2346577290091983,0.23333821657015275,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p005-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p01,0,RMSE,rmse,lower,0.23515957801466283,-0.23515957801466283,0.23255229143999848,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p01-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p001,0,RMSE,rmse,lower,0.234848704054223,-0.234848704054223,0.23316213384908455,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p001-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p005,0,RMSE,rmse,lower,0.23504563388193753,-0.23504563388193753,0.2337165723322532,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p005-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p01,0,RMSE,rmse,lower,0.2351138707244049,-0.2351138707244049,0.23346224116260836,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p01-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p001,0,RMSE,rmse,lower,0.2350451108570483,-0.2350451108570483,0.23328519811438406,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p001-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p005,0,RMSE,rmse,lower,0.23490809371963584,-0.23490809371963584,0.23175764135614477,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p005-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p01,0,RMSE,rmse,lower,0.23465687967262416,-0.23465687967262416,0.23108455049001858,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p01-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p001,0,RMSE,rmse,lower,0.23496993373476788,-0.23496993373476788,0.23346087685954336,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p001-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p005,0,RMSE,rmse,lower,0.23492098632735753,-0.23492098632735753,0.2411043186306478,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p005-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p01,0,RMSE,rmse,lower,0.23479386888101392,-0.23479386888101392,0.23419678588244902,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p01-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p001,0,RMSE,rmse,lower,0.23544191074619497,-0.23544191074619497,0.23248160420213973,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p001-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p005,0,RMSE,rmse,lower,0.23458071263974148,-0.23458071263974148,0.23412343670770672,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p005-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p01,0,RMSE,rmse,lower,0.23471617503170125,-0.23471617503170125,0.23405948700639934,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p01-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p001,0,RMSE,rmse,lower,0.23558367762733304,-0.23558367762733304,0.2306278293973631,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p001-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p005,0,RMSE,rmse,lower,0.23476767229656906,-0.23476767229656906,0.23170820829575262,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p005-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p01,0,RMSE,rmse,lower,0.2351196777064841,-0.2351196777064841,0.23285365668962787,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p01-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p001,0,RMSE,rmse,lower,0.23556640917246102,-0.23556640917246102,0.23394963288342588,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p001-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p005,0,RMSE,rmse,lower,0.23502662197192856,-0.23502662197192856,0.23487478850600307,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p005-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p01,0,RMSE,rmse,lower,0.2349593190227014,-0.2349593190227014,0.23427020750192581,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p01-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p001,0,RMSE,rmse,lower,0.23522375918391628,-0.23522375918391628,0.23361716934488255,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p001-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p005,0,RMSE,rmse,lower,0.23474049679651998,-0.23474049679651998,0.23271906309028628,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p005-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p01,0,RMSE,rmse,lower,0.23523273079765322,-0.23523273079765322,0.231476592513396,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p01-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p001,0,RMSE,rmse,lower,0.23522175576712184,-0.23522175576712184,0.2392933877459825,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p001-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p005,0,RMSE,rmse,lower,0.2351262925795415,-0.2351262925795415,0.23336620188006213,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p005-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p01,0,RMSE,rmse,lower,0.23509613200143026,-0.23509613200143026,0.23336054282826887,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p01-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p001,0,RMSE,rmse,lower,0.2346724530004704,-0.2346724530004704,0.23472572946107031,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p001-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p005,0,RMSE,rmse,lower,0.23514965307924723,-0.23514965307924723,0.23159657135010447,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p005-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p01,0,RMSE,rmse,lower,0.23526017405140137,-0.23526017405140137,0.23237540477719482,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p01-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p001,0,RMSE,rmse,lower,0.2350854992177386,-0.2350854992177386,0.23319390245153368,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p001-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p005,0,RMSE,rmse,lower,0.23460196790457225,-0.23460196790457225,0.23160625447220415,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p005-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p01,0,RMSE,rmse,lower,0.23525231195767085,-0.23525231195767085,0.23376059267196012,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p01-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p001,0,RMSE,rmse,lower,0.2352393029012561,-0.2352393029012561,0.23369680670218232,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p001-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p005,0,RMSE,rmse,lower,0.23530708779168627,-0.23530708779168627,0.23121293389473055,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p005-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p01,0,RMSE,rmse,lower,0.234867262459557,-0.234867262459557,0.23225197071547168,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p01-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p001,0,RMSE,rmse,lower,0.23505533340532211,-0.23505533340532211,0.2334740169208309,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p001-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p005,0,RMSE,rmse,lower,0.23498608081258596,-0.23498608081258596,0.23378104608060551,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p005-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p01,0,RMSE,rmse,lower,0.23478007280066537,-0.23478007280066537,0.2351072792694515,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p01-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0p0001_l0p02,0,RMSE,rmse,lower,0.23785345573254446,-0.23785345573254446,0.23747813004889556,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p02-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0p0001_l0p02,0,RMSE,rmse,lower,0.23340410361335404,-0.23340410361335404,0.2382911617531396,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p02-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0p001_l0p02,0,RMSE,rmse,lower,0.23785345573254446,-0.23785345573254446,0.23747813004889556,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p02-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0p001_l0p02,0,RMSE,rmse,lower,0.23340410361335404,-0.23340410361335404,0.2382911617531396,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p02-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n1em05_l0p02,0,RMSE,rmse,lower,0.23785345573254446,-0.23785345573254446,0.23747813004889556,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p02-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0_l0p02,0,RMSE,rmse,lower,0.23785345573254446,-0.23785345573254446,0.23747813004889556,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p02-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n1em05_l0p02,0,RMSE,rmse,lower,0.23340410361335404,-0.23340410361335404,0.2382911617531396,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p02-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0_l0p02,0,RMSE,rmse,lower,0.23340410361335404,-0.23340410361335404,0.2382911617531396,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p02-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n1em05_l0p005,0,RMSE,rmse,lower,0.2390328497980867,-0.2390328497980867,0.24063334636332392,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p005-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n1em05_l0p01,0,RMSE,rmse,lower,0.23909857764435258,-0.23909857764435258,0.24513721764174634,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p01-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n1em05_l0p005,0,RMSE,rmse,lower,0.2337388066681912,-0.2337388066681912,0.2344355028261131,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p005-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0p001_l0p001,0,RMSE,rmse,lower,0.2390544572113064,-0.2390544572113064,0.25549556988031324,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p001-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n1em05_l0p01,0,RMSE,rmse,lower,0.23356265910507878,-0.23356265910507878,0.23721029814115205,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p01-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0p001_l0p001,0,RMSE,rmse,lower,0.23362806029940672,-0.23362806029940672,0.23621330151542805,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p001-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0p001_l0p01,0,RMSE,rmse,lower,0.23909857764435258,-0.23909857764435258,0.24513721764174634,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p01-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0p001_l0p01,0,RMSE,rmse,lower,0.23356265910507878,-0.23356265910507878,0.23721029814115205,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p01-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0_l0p001,0,RMSE,rmse,lower,0.2390544572113064,-0.2390544572113064,0.25549556988031324,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p001-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0p001_l0p005,0,RMSE,rmse,lower,0.2390328497980867,-0.2390328497980867,0.24063334636332392,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p005-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n1em05_l0p001,0,RMSE,rmse,lower,0.2390544572113064,-0.2390544572113064,0.25549556988031324,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p001-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0p001_l0p005,0,RMSE,rmse,lower,0.2337388066681912,-0.2337388066681912,0.2344355028261131,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p005-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0_l0p001,0,RMSE,rmse,lower,0.23362806029940672,-0.23362806029940672,0.23621330151542805,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p001-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n1em05_l0p001,0,RMSE,rmse,lower,0.23362806029940672,-0.23362806029940672,0.23621330151542805,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p001-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0p0001_l0p001,0,RMSE,rmse,lower,0.2390544572113064,-0.2390544572113064,0.25549556988031324,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p001-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0p0001_l0p01,0,RMSE,rmse,lower,0.23909857764435258,-0.23909857764435258,0.24513721764174634,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p01-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0_l0p005,0,RMSE,rmse,lower,0.2390328497980867,-0.2390328497980867,0.24063334636332392,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p005-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0p0001_l0p001,0,RMSE,rmse,lower,0.23362806029940672,-0.23362806029940672,0.23621330151542805,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p001-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0_l0p005,0,RMSE,rmse,lower,0.2337388066681912,-0.2337388066681912,0.2344355028261131,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p005-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0p0001_l0p01,0,RMSE,rmse,lower,0.23356265910507878,-0.23356265910507878,0.23721029814115205,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p01-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0_l0p01,0,RMSE,rmse,lower,0.23909857764435258,-0.23909857764435258,0.24513721764174634,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p01-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0p0001_l0p005,0,RMSE,rmse,lower,0.2390328497980867,-0.2390328497980867,0.24063334636332392,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p005-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0_l0p01,0,RMSE,rmse,lower,0.23356265910507878,-0.23356265910507878,0.23721029814115205,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p01-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0p0001_l0p005,0,RMSE,rmse,lower,0.2337388066681912,-0.2337388066681912,0.2344355028261131,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p005-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0_l0p02,0,RMSE,rmse,lower,0.23785555444789067,-0.23785555444789067,0.23748897727684737,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p02-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n1em05_l0p005,0,RMSE,rmse,lower,0.23907342929415865,-0.23907342929415865,0.23310698183908637,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p005-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n1em05_l0p01,0,RMSE,rmse,lower,0.2376162677316268,-0.2376162677316268,0.23517437354931303,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p01-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0_l0p02,0,RMSE,rmse,lower,0.23340542834677785,-0.23340542834677785,0.23829137280333715,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p02-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n1em05_l0p005,0,RMSE,rmse,lower,0.2332399859921054,-0.2332399859921054,0.2326072306387775,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p005-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n1em05_l0p01,0,RMSE,rmse,lower,0.2335073783441898,-0.2335073783441898,0.2329070852784964,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p01-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0p001_l0p01,0,RMSE,rmse,lower,0.23798534123823967,-0.23798534123823967,0.2395424764805503,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p01-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0p001_l0p01,0,RMSE,rmse,lower,0.23320259273091312,-0.23320259273091312,0.23641549221562166,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p01-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0_l0p001,0,RMSE,rmse,lower,0.23905700508983188,-0.23905700508983188,0.2554892709638668,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p001-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0_l0p001,0,RMSE,rmse,lower,0.23363252496566134,-0.23363252496566134,0.2362188764537236,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p001-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n1em05_l0p001,0,RMSE,rmse,lower,0.23716996532706527,-0.23716996532706527,0.23470593771148443,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p001-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n1em05_l0p001,0,RMSE,rmse,lower,0.23379921122284117,-0.23379921122284117,0.23898482806189927,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p001-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0p001_l0p005,0,RMSE,rmse,lower,0.23885129531058336,-0.23885129531058336,0.24654667666950852,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p005-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0p001_l0p005,0,RMSE,rmse,lower,0.23374365968826152,-0.23374365968826152,0.23258765102786882,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p005-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n1em05_l0p02,0,RMSE,rmse,lower,0.23720560242248867,-0.23720560242248867,0.23787240614757268,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p02-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n1em05_l0p02,0,RMSE,rmse,lower,0.23350963577191214,-0.23350963577191214,0.2393402580368093,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p02-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0_l0p005,0,RMSE,rmse,lower,0.23901989843968774,-0.23901989843968774,0.24063495640158664,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p005-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0_l0p005,0,RMSE,rmse,lower,0.23373660723675202,-0.23373660723675202,0.23443243594536417,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p005-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0p0001_l0p01,0,RMSE,rmse,lower,0.23912916812165463,-0.23912916812165463,0.23873190398066874,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p01-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0p0001_l0p01,0,RMSE,rmse,lower,0.2333667286677465,-0.2333667286677465,0.2358601642328826,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p01-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0_l0p01,0,RMSE,rmse,lower,0.239085684381371,-0.239085684381371,0.24513387433279307,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p01-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0p0001_l0p005,0,RMSE,rmse,lower,0.23811064537966578,-0.23811064537966578,0.24336730552230365,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p005-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0_l0p01,0,RMSE,rmse,lower,0.23356035433981295,-0.23356035433981295,0.2372102431750833,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p01-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0p0001_l0p005,0,RMSE,rmse,lower,0.23317303008072252,-0.23317303008072252,0.23484075680143124,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p005-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0p0001_l0p001,0,RMSE,rmse,lower,0.23798803361544055,-0.23798803361544055,0.23971064008247925,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p001-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0p0001_l0p001,0,RMSE,rmse,lower,0.23342205869781213,-0.23342205869781213,0.23945135735112927,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p001-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0p001_l0p02,0,RMSE,rmse,lower,0.23771204736707321,-0.23771204736707321,0.24308994802985293,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p02-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0p001_l0p02,0,RMSE,rmse,lower,0.2334750859643239,-0.2334750859643239,0.23518646749461922,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p02-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0p0001_l0p02,0,RMSE,rmse,lower,0.23792787050471845,-0.23792787050471845,0.24466013756612576,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p02-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0p0001_l0p02,0,RMSE,rmse,lower,0.23322026782389782,-0.23322026782389782,0.23659461735657236,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p02-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0p001_l0p001,0,RMSE,rmse,lower,0.24061645579340615,-0.24061645579340615,0.2630298852149786,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p001-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0p001_l0p001,0,RMSE,rmse,lower,0.23376432971584016,-0.23376432971584016,0.24301364183447563,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p001-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0_l0p02,0,RMSE,rmse,lower,0.23785555444789067,-0.23785555444789067,0.23748897727684737,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p02-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0_l0p02,0,RMSE,rmse,lower,0.23340542834677785,-0.23340542834677785,0.23829137280333715,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p02-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n1em05_l0p01,0,RMSE,rmse,lower,0.2383569065742623,-0.2383569065742623,0.244620972160157,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p01-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n1em05_l0p01,0,RMSE,rmse,lower,0.2334130414280393,-0.2334130414280393,0.23649611689361716,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p01-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n1em05_l0p02,0,RMSE,rmse,lower,0.23747199640064248,-0.23747199640064248,0.24725484176674034,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p02-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n1em05_l0p02,0,RMSE,rmse,lower,0.23366474775887086,-0.23366474775887086,0.23646780882908092,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p02-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0p0001_l0p005,0,RMSE,rmse,lower,0.23814119850842008,-0.23814119850842008,0.2663355327951239,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p005-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0p0001_l0p005,0,RMSE,rmse,lower,0.23343416363529057,-0.23343416363529057,0.234433762813203,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p005-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0p0001_l0p02,0,RMSE,rmse,lower,0.23869658059774151,-0.23869658059774151,0.24052100411353006,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p02-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0p0001_l0p02,0,RMSE,rmse,lower,0.23353358071122673,-0.23353358071122673,0.23738632838570875,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p02-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0_l0p01,0,RMSE,rmse,lower,0.239085684381371,-0.239085684381371,0.24513387433279307,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p01-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0p0001_l0p01,0,RMSE,rmse,lower,0.23744052562658574,-0.23744052562658574,0.2382342886377602,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p01-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0_l0p01,0,RMSE,rmse,lower,0.23356035433981295,-0.23356035433981295,0.2372102431750833,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p01-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0p0001_l0p01,0,RMSE,rmse,lower,0.23321172995482362,-0.23321172995482362,0.234184776040884,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p01-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0_l0p005,0,RMSE,rmse,lower,0.23901989843968774,-0.23901989843968774,0.24063495640158664,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p005-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0_l0p005,0,RMSE,rmse,lower,0.23373660723675202,-0.23373660723675202,0.23443243594536417,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p005-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0p001_l0p02,0,RMSE,rmse,lower,0.23843109390982442,-0.23843109390982442,0.24318669684901736,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p02-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0p001_l0p02,0,RMSE,rmse,lower,0.2334736499346226,-0.2334736499346226,0.23433651608184083,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p02-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0_l0p001,0,RMSE,rmse,lower,0.23905700508983188,-0.23905700508983188,0.2554892709638668,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p001-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0_l0p001,0,RMSE,rmse,lower,0.23363252496566134,-0.23363252496566134,0.2362188764537236,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p001-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0p001_l0p01,0,RMSE,rmse,lower,0.23850314174188256,-0.23850314174188256,0.23244720609452174,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p01-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0p001_l0p01,0,RMSE,rmse,lower,0.233651251716616,-0.233651251716616,0.2427291888102559,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p01-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n1em05_l0p001,0,RMSE,rmse,lower,0.2388823307131178,-0.2388823307131178,0.25624351202497053,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p001-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n1em05_l0p001,0,RMSE,rmse,lower,0.23408501487488748,-0.23408501487488748,0.23576018763280351,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p001-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n1em05_l0p005,0,RMSE,rmse,lower,0.23844761590759667,-0.23844761590759667,0.2416540932193429,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p005-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n1em05_l0p005,0,RMSE,rmse,lower,0.23314779382394482,-0.23314779382394482,0.23448866631035667,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p005-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0p0001_l0p001,0,RMSE,rmse,lower,0.23779726122481318,-0.23779726122481318,0.24153146027417532,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p001-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0p0001_l0p001,0,RMSE,rmse,lower,0.23334639858951206,-0.23334639858951206,0.23179329091828219,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p001-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0p001_l0p005,0,RMSE,rmse,lower,0.239770922958097,-0.239770922958097,0.23915230117801708,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p005-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0p001_l0p005,0,RMSE,rmse,lower,0.23367947054972119,-0.23367947054972119,0.23861273393383206,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p005-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0p001_l0p001,0,RMSE,rmse,lower,0.23828438459742213,-0.23828438459742213,0.2530485190714714,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p001-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0p001_l0p001,0,RMSE,rmse,lower,0.23357213312617114,-0.23357213312617114,0.23844724876452805,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p001-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0_rho0p00025,0,RMSE,rmse,lower,0.23857661998288315,-0.23857661998288315,0.2544202590204169,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p00025-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0_rho0p00025,0,RMSE,rmse,lower,0.23320292020753594,-0.23320292020753594,0.23031042609193297,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p00025-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n1em05_rho0p00025,0,RMSE,rmse,lower,0.23857661998288315,-0.23857661998288315,0.2544202590204169,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p00025-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n1em05_rho0p00025,0,RMSE,rmse,lower,0.23320292020753594,-0.23320292020753594,0.23031042609193297,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p00025-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0p0001_rho0p00025,0,RMSE,rmse,lower,0.23857661998288315,-0.23857661998288315,0.2544202590204169,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p00025-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0p0001_rho0p00025,0,RMSE,rmse,lower,0.23320292020753594,-0.23320292020753594,0.23031042609193297,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p00025-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0_rho0p0005,0,RMSE,rmse,lower,0.2390755017179478,-0.2390755017179478,0.25168892959044736,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0005-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0_rho0p0005,0,RMSE,rmse,lower,0.2333487134502395,-0.2333487134502395,0.2416932153437863,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0005-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0p001_rho0p00025,0,RMSE,rmse,lower,0.23857661998288315,-0.23857661998288315,0.2544202590204169,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p00025-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0p001_rho0p00025,0,RMSE,rmse,lower,0.23320292020753594,-0.23320292020753594,0.23031042609193297,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p00025-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0_rho0p0025,0,RMSE,rmse,lower,0.2377136850270048,-0.2377136850270048,0.262345673404661,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0025-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0_rho0p0025,0,RMSE,rmse,lower,0.23295802298295287,-0.23295802298295287,0.23903366020873654,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0025-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n1em05_rho0p0005,0,RMSE,rmse,lower,0.2390755017179478,-0.2390755017179478,0.25168892959044736,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0005-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n1em05_rho0p0005,0,RMSE,rmse,lower,0.2333487134502395,-0.2333487134502395,0.2416932153437863,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0005-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n1em05_rho0p0025,0,RMSE,rmse,lower,0.2377136850270048,-0.2377136850270048,0.262345673404661,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0025-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n1em05_rho0p0025,0,RMSE,rmse,lower,0.23295802298295287,-0.23295802298295287,0.23903366020873654,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0025-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0_rho0p001,0,RMSE,rmse,lower,0.23949031043262045,-0.23949031043262045,0.24155888192007524,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p001-best-head-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0_rho0p001,0,RMSE,rmse,lower,0.23316146280351166,-0.23316146280351166,0.23452524296935331,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p001-greedy-heads-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n1em05_rho0p001,0,RMSE,rmse,lower,0.23949031043262045,-0.23949031043262045,0.24155888192007524,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p001-best-head-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0p0001_rho0p0005,0,RMSE,rmse,lower,0.2390755017179478,-0.2390755017179478,0.25168892959044736,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0005-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0p0001_rho0p0005,0,RMSE,rmse,lower,0.2333487134502395,-0.2333487134502395,0.2416932153437863,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0005-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n1em05_rho0p001,0,RMSE,rmse,lower,0.23316146280351166,-0.23316146280351166,0.23452524296935331,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p001-greedy-heads-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0p0001_rho0p0025,0,RMSE,rmse,lower,0.2377136850270048,-0.2377136850270048,0.262345673404661,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0025-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0p0001_rho0p0025,0,RMSE,rmse,lower,0.23295802298295287,-0.23295802298295287,0.23903366020873654,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0025-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0p001_rho0p0005,0,RMSE,rmse,lower,0.2390755017179478,-0.2390755017179478,0.25168892959044736,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0005-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0p0001_rho0p001,0,RMSE,rmse,lower,0.23949031043262045,-0.23949031043262045,0.24155888192007524,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p001-best-head-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0p001_rho0p0005,0,RMSE,rmse,lower,0.2333487134502395,-0.2333487134502395,0.2416932153437863,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0005-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0p0001_rho0p001,0,RMSE,rmse,lower,0.23316146280351166,-0.23316146280351166,0.23452524296935331,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p001-greedy-heads-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0p001_rho0p0025,0,RMSE,rmse,lower,0.2377136850270048,-0.2377136850270048,0.262345673404661,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0025-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0p001_rho0p0025,0,RMSE,rmse,lower,0.23295802298295287,-0.23295802298295287,0.23903366020873654,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0025-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0p001_rho0p001,0,RMSE,rmse,lower,0.23949031043262045,-0.23949031043262045,0.24155888192007524,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p001-best-head-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0p001_rho0p001,0,RMSE,rmse,lower,0.23316146280351166,-0.23316146280351166,0.23452524296935331,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p001-greedy-heads-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n1em05_rho0p00025,0,RMSE,rmse,lower,0.23736952064793038,-0.23736952064793038,0.24275107338262392,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p00025-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n1em05_rho0p001,0,RMSE,rmse,lower,0.23826622524732466,-0.23826622524732466,0.2354789879616611,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p001-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n1em05_rho0p00025,0,RMSE,rmse,lower,0.23322604208950684,-0.23322604208950684,0.23242508060584816,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p00025-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n1em05_rho0p001,0,RMSE,rmse,lower,0.23311639447730767,-0.23311639447730767,0.2316950082866651,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p001-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0_rho0p0005,0,RMSE,rmse,lower,0.2383891939899224,-0.2383891939899224,0.24401096715766643,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0005-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0_rho0p0005,0,RMSE,rmse,lower,0.23289370528688721,-0.23289370528688721,0.2404508003490849,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0005-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n1em05_rho0p0025,0,RMSE,rmse,lower,0.23826509952531,-0.23826509952531,0.23646242093377673,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0025-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0_rho0p00025,0,RMSE,rmse,lower,0.24006717570021247,-0.24006717570021247,0.24244266573296758,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p00025-best-head-evaluation/0,False,962 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n1em05_rho0p0025,0,RMSE,rmse,lower,0.23303048406462032,-0.23303048406462032,0.23273007610764737,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0025-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0_rho0p00025,0,RMSE,rmse,lower,0.23372927564914306,-0.23372927564914306,0.2350238164160606,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p00025-greedy-heads-evaluation/0,False,962 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0p0001_rho0p0025,0,RMSE,rmse,lower,0.23800908632875115,-0.23800908632875115,0.2395995521584212,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0025-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0p0001_rho0p0025,0,RMSE,rmse,lower,0.2331158511435021,-0.2331158511435021,0.23340534056347034,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0025-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0_rho0p001,0,RMSE,rmse,lower,0.23897535038319878,-0.23897535038319878,0.24383994851478993,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p001-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0_rho0p001,0,RMSE,rmse,lower,0.23343394819363691,-0.23343394819363691,0.2365000863679966,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p001-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0p0001_rho0p001,0,RMSE,rmse,lower,0.23848516306571665,-0.23848516306571665,0.24223068067706838,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p001-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0p0001_rho0p001,0,RMSE,rmse,lower,0.23318228828211082,-0.23318228828211082,0.23425661912073115,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p001-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0p001_rho0p00025,0,RMSE,rmse,lower,0.23803283687571922,-0.23803283687571922,0.273329277420384,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p00025-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0p001_rho0p00025,0,RMSE,rmse,lower,0.23289638454580067,-0.23289638454580067,0.23281509730791192,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p00025-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n1em05_rho0p0005,0,RMSE,rmse,lower,0.2393372540058719,-0.2393372540058719,0.2906284957593463,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0005-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n1em05_rho0p0005,0,RMSE,rmse,lower,0.23272791516423366,-0.23272791516423366,0.23258942887652473,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0005-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0_rho0p0025,0,RMSE,rmse,lower,0.2401098998840954,-0.2401098998840954,0.24179294965081416,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0025-best-head-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0_rho0p0025,0,RMSE,rmse,lower,0.23332136469753356,-0.23332136469753356,0.2364625390906921,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0025-greedy-heads-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0p0001_rho0p00025,0,RMSE,rmse,lower,0.23704860354665752,-0.23704860354665752,0.23874426241915847,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p00025-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0p0001_rho0p00025,0,RMSE,rmse,lower,0.23324589552748487,-0.23324589552748487,0.23661615616090015,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p00025-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0p001_rho0p0025,0,RMSE,rmse,lower,0.23784205343302714,-0.23784205343302714,0.23492100218498793,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0025-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0p001_rho0p0025,0,RMSE,rmse,lower,0.23278002032353212,-0.23278002032353212,0.23227273339288673,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0025-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0p0001_rho0p0005,0,RMSE,rmse,lower,0.23835052202476453,-0.23835052202476453,0.23699550345285816,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0005-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0p0001_rho0p0005,0,RMSE,rmse,lower,0.2333328522027255,-0.2333328522027255,0.238922935773872,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0005-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0p001_rho0p0005,0,RMSE,rmse,lower,0.24035605783658937,-0.24035605783658937,0.2779356858657707,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0005-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0p001_rho0p0005,0,RMSE,rmse,lower,0.23367927924707155,-0.23367927924707155,0.24524323007832846,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0005-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0p001_rho0p001,0,RMSE,rmse,lower,0.23789169956459427,-0.23789169956459427,0.23950414627424274,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p001-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0p001_rho0p001,0,RMSE,rmse,lower,0.23367845823808855,-0.23367845823808855,0.24371457881808845,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p001-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0_rho0p00025,0,RMSE,rmse,lower,0.23818537080734506,-0.23818537080734506,0.23894844295090592,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p00025-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0_rho0p00025,0,RMSE,rmse,lower,0.2337721462391934,-0.2337721462391934,0.2337674053617211,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p00025-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0_rho0p0005,0,RMSE,rmse,lower,0.23826995416351013,-0.23826995416351013,0.23722919786139868,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0005-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0_rho0p0005,0,RMSE,rmse,lower,0.23341469329024547,-0.23341469329024547,0.23271509315390915,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0005-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n1em05_rho0p0005,0,RMSE,rmse,lower,0.23743311228743408,-0.23743311228743408,0.24918858157196577,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0005-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n1em05_rho0p0005,0,RMSE,rmse,lower,0.23308539046397073,-0.23308539046397073,0.23474214725238218,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0005-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0_rho0p0025,0,RMSE,rmse,lower,0.23785759049732064,-0.23785759049732064,0.23589133259357026,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0025-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0_rho0p0025,0,RMSE,rmse,lower,0.23269480219963232,-0.23269480219963232,0.23239933834285403,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0025-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0p0001_rho0p0005,0,RMSE,rmse,lower,0.2382764815807058,-0.2382764815807058,0.24121307650092602,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0005-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0p0001_rho0p0005,0,RMSE,rmse,lower,0.23269733966625106,-0.23269733966625106,0.23483414974881817,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0005-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0p001_rho0p0005,0,RMSE,rmse,lower,0.23686825686043878,-0.23686825686043878,0.24350702848855615,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0005-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0p001_rho0p0005,0,RMSE,rmse,lower,0.23313960483456422,-0.23313960483456422,0.23741637847101169,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0005-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0p001_rho0p001,0,RMSE,rmse,lower,0.23779476251258524,-0.23779476251258524,0.2405214145563063,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p001-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0p001_rho0p001,0,RMSE,rmse,lower,0.2333871447652357,-0.2333871447652357,0.2345517842823023,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p001-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0_rho0p001,0,RMSE,rmse,lower,0.23860257792551076,-0.23860257792551076,0.30043019206533295,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p001-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0_rho0p001,0,RMSE,rmse,lower,0.2336154232368663,-0.2336154232368663,0.24064628828924506,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p001-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n1em05_rho0p00025,0,RMSE,rmse,lower,0.23842471915921798,-0.23842471915921798,0.34030246469362907,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p00025-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n1em05_rho0p00025,0,RMSE,rmse,lower,0.23367244804564585,-0.23367244804564585,0.25122008031651116,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p00025-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n1em05_rho0p0025,0,RMSE,rmse,lower,0.2380240960109698,-0.2380240960109698,0.23356247568168645,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0025-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n1em05_rho0p0025,0,RMSE,rmse,lower,0.23277917213756988,-0.23277917213756988,0.2432190859636802,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0025-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n1em05_rho0p001,0,RMSE,rmse,lower,0.23707897142181117,-0.23707897142181117,0.23966200804896912,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p001-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0p0001_rho0p0025,0,RMSE,rmse,lower,0.2393572230996531,-0.2393572230996531,0.23868415725996342,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0025-best-head-evaluation/0,False,740 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n1em05_rho0p001,0,RMSE,rmse,lower,0.23314020403866242,-0.23314020403866242,0.23450263045082137,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p001-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0p0001_rho0p0025,0,RMSE,rmse,lower,0.23411584672739014,-0.23411584672739014,0.23627903325948496,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0025-greedy-heads-evaluation/0,False,740 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0p0001_rho0p00025,0,RMSE,rmse,lower,0.23957945554469914,-0.23957945554469914,0.2496147390992142,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p00025-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0p0001_rho0p00025,0,RMSE,rmse,lower,0.23353144315489308,-0.23353144315489308,0.2478765142940872,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p00025-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0p001_rho0p00025,0,RMSE,rmse,lower,0.23889975711739106,-0.23889975711739106,0.23704205803317221,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p00025-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0p001_rho0p00025,0,RMSE,rmse,lower,0.23338687341364966,-0.23338687341364966,0.23796418466778702,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p00025-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0p0001_rho0p001,0,RMSE,rmse,lower,0.2387734473283714,-0.2387734473283714,0.23954579674332094,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p001-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0p0001_rho0p001,0,RMSE,rmse,lower,0.23413401771618858,-0.23413401771618858,0.23885206734563352,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p001-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0p001_rho0p0025,0,RMSE,rmse,lower,0.2389336160662929,-0.2389336160662929,0.2358168359752649,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0025-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0p001_rho0p0025,0,RMSE,rmse,lower,0.23389711137908714,-0.23389711137908714,0.2407960992475978,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0025-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p001,0,RMSE,rmse,lower,0.23897181954001903,-0.23897181954001903,0.23496733361066027,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p001-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p001,0,RMSE,rmse,lower,0.2341784288692296,-0.2341784288692296,0.23454983071534766,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p001-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p01,0,RMSE,rmse,lower,0.23781902006036262,-0.23781902006036262,0.2470883298795412,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p01-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p01,0,RMSE,rmse,lower,0.2335463179080623,-0.2335463179080623,0.23522276143697368,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p01-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p01,0,RMSE,rmse,lower,0.23756948853650636,-0.23756948853650636,0.24224213014777235,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p01-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p01,0,RMSE,rmse,lower,0.23291770554949479,-0.23291770554949479,0.23458472246540202,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p01-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p001,0,RMSE,rmse,lower,0.23862914188251422,-0.23862914188251422,0.23553986357063933,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p001-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p001,0,RMSE,rmse,lower,0.23305490182627153,-0.23305490182627153,0.23732565171059752,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p001-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p005,0,RMSE,rmse,lower,0.23918568828057105,-0.23918568828057105,0.25293341379845147,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p005-best-head-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p005,0,RMSE,rmse,lower,0.2331196304833006,-0.2331196304833006,0.23280496841127477,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p005-greedy-heads-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p005,0,RMSE,rmse,lower,0.2372450969584346,-0.2372450969584346,0.23726047684878263,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p005-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001,0,RMSE,rmse,lower,0.23560357747729638,-0.23560357747729638,0.2389191702722735,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001,0,RMSE,rmse,lower,0.2324412121462238,-0.2324412121462238,0.23820740696857753,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p005,0,RMSE,rmse,lower,0.2328726380397863,-0.2328726380397863,0.23808236498886684,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p005-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p01,0,RMSE,rmse,lower,0.23656618722639788,-0.23656618722639788,0.2443526301182123,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p01-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p01,0,RMSE,rmse,lower,0.23334387616340208,-0.23334387616340208,0.2380340654227449,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p01-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p001,0,RMSE,rmse,lower,0.23897181954001903,-0.23897181954001903,0.23496733361066027,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p001-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p005,0,RMSE,rmse,lower,0.24058462987099405,-0.24058462987099405,0.2859647081281448,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p005-best-head-evaluation/0,False,1184 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p001,0,RMSE,rmse,lower,0.2341784288692296,-0.2341784288692296,0.23454983071534766,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p001-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p005,0,RMSE,rmse,lower,0.23316436266516308,-0.23316436266516308,0.23171057167720172,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p005-greedy-heads-evaluation/0,False,1184 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p01,0,RMSE,rmse,lower,0.23781902006036262,-0.23781902006036262,0.2470883298795412,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p01-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p01,0,RMSE,rmse,lower,0.2335463179080623,-0.2335463179080623,0.23522276143697368,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p01-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p01,0,RMSE,rmse,lower,0.23756948853650636,-0.23756948853650636,0.24224213014777235,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p01-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p01,0,RMSE,rmse,lower,0.23291770554949479,-0.23291770554949479,0.23458472246540202,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p01-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001,0,RMSE,rmse,lower,0.23560357747729638,-0.23560357747729638,0.2389191702722735,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p001,0,RMSE,rmse,lower,0.23862914188251422,-0.23862914188251422,0.23553986357063933,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p001-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001,0,RMSE,rmse,lower,0.2324412121462238,-0.2324412121462238,0.23820740696857753,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p001,0,RMSE,rmse,lower,0.23305490182627153,-0.23305490182627153,0.23732565171059752,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p001-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p005,0,RMSE,rmse,lower,0.23918568828057105,-0.23918568828057105,0.25293341379845147,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p005-best-head-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p005,0,RMSE,rmse,lower,0.2331196304833006,-0.2331196304833006,0.23280496841127477,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p005-greedy-heads-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p005,0,RMSE,rmse,lower,0.24058462987099405,-0.24058462987099405,0.2859647081281448,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p005-best-head-evaluation/0,False,1184 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p005,0,RMSE,rmse,lower,0.23316436266516308,-0.23316436266516308,0.23171057167720172,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p005-greedy-heads-evaluation/0,False,1184 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p005,0,RMSE,rmse,lower,0.2372450969584346,-0.2372450969584346,0.23726047684878263,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p005-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p005,0,RMSE,rmse,lower,0.2328726380397863,-0.2328726380397863,0.23808236498886684,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p005-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p01,0,RMSE,rmse,lower,0.23656618722639788,-0.23656618722639788,0.2443526301182123,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p01-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p01,0,RMSE,rmse,lower,0.23334387616340208,-0.23334387616340208,0.2380340654227449,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p01-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p001,0,RMSE,rmse,lower,0.23897181954001903,-0.23897181954001903,0.23496733361066027,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p001-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p001,0,RMSE,rmse,lower,0.2341784288692296,-0.2341784288692296,0.23454983071534766,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p001-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p01,0,RMSE,rmse,lower,0.23781902006036262,-0.23781902006036262,0.2470883298795412,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p01-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p01,0,RMSE,rmse,lower,0.2335463179080623,-0.2335463179080623,0.23522276143697368,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p01-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p01,0,RMSE,rmse,lower,0.23756948853650636,-0.23756948853650636,0.24224213014777235,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p01-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p01,0,RMSE,rmse,lower,0.23291770554949479,-0.23291770554949479,0.23458472246540202,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p01-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p01,0,RMSE,rmse,lower,0.23656618722639788,-0.23656618722639788,0.2443526301182123,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p01-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p01,0,RMSE,rmse,lower,0.23334387616340208,-0.23334387616340208,0.2380340654227449,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p01-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p001,0,RMSE,rmse,lower,0.23862914188251422,-0.23862914188251422,0.23553986357063933,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p001-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p001,0,RMSE,rmse,lower,0.23560357747729638,-0.23560357747729638,0.2389191702722735,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p001-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p001,0,RMSE,rmse,lower,0.23305490182627153,-0.23305490182627153,0.23732565171059752,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p001-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p001,0,RMSE,rmse,lower,0.2324412121462238,-0.2324412121462238,0.23820740696857753,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p001-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p005,0,RMSE,rmse,lower,0.2372450969584346,-0.2372450969584346,0.23726047684878263,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p005-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p005,0,RMSE,rmse,lower,0.2328726380397863,-0.2328726380397863,0.23808236498886684,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p005-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p005,0,RMSE,rmse,lower,0.23918568828057105,-0.23918568828057105,0.25293341379845147,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p005-best-head-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p005,0,RMSE,rmse,lower,0.2331196304833006,-0.2331196304833006,0.23280496841127477,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p005-greedy-heads-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p005,0,RMSE,rmse,lower,0.24058462987099405,-0.24058462987099405,0.2859647081281448,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p005-best-head-evaluation/0,False,1184 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p005,0,RMSE,rmse,lower,0.23316436266516308,-0.23316436266516308,0.23171057167720172,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p005-greedy-heads-evaluation/0,False,1184 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p005,0,RMSE,rmse,lower,0.23790953518728727,-0.23790953518728727,0.23821708722105353,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p005-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p005,0,RMSE,rmse,lower,0.23395800051802765,-0.23395800051802765,0.23212124110153423,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p005-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p01,0,RMSE,rmse,lower,0.23752709943108333,-0.23752709943108333,0.23270363118336854,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p01-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p01,0,RMSE,rmse,lower,0.23302229095267976,-0.23302229095267976,0.2330139696533979,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p01-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p001,0,RMSE,rmse,lower,0.23774841016444426,-0.23774841016444426,0.24035398870465244,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p001-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p001,0,RMSE,rmse,lower,0.23360989779537025,-0.23360989779537025,0.24433293207334258,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p001-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p001,0,RMSE,rmse,lower,0.23843676541966466,-0.23843676541966466,0.2470445431022663,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p001-best-head-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p001,0,RMSE,rmse,lower,0.23360278549070665,-0.23360278549070665,0.23177048418825977,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p001-greedy-heads-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p005,0,RMSE,rmse,lower,0.23613982935624828,-0.23613982935624828,0.23869431760070778,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p005-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p005,0,RMSE,rmse,lower,0.23307964467970407,-0.23307964467970407,0.23920139503937368,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p005-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p001,0,RMSE,rmse,lower,0.23604388522704514,-0.23604388522704514,0.2413116661749965,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p001-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p001,0,RMSE,rmse,lower,0.23298097729336806,-0.23298097729336806,0.23673558379896892,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p001-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p01,0,RMSE,rmse,lower,0.23924971586268343,-0.23924971586268343,0.23943478014622727,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p01-best-head-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p01,0,RMSE,rmse,lower,0.233123281922542,-0.233123281922542,0.2324621021496061,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p01-greedy-heads-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p005,0,RMSE,rmse,lower,0.2369960693297996,-0.2369960693297996,0.2449247661219165,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p005-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p005,0,RMSE,rmse,lower,0.23320698567170814,-0.23320698567170814,0.24367480279619608,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p005-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p01,0,RMSE,rmse,lower,0.23723977382385666,-0.23723977382385666,0.24324646285162105,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p01-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p01,0,RMSE,rmse,lower,0.23277517922062366,-0.23277517922062366,0.239397685367944,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p01-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p001,0,RMSE,rmse,lower,0.2382472279758646,-0.2382472279758646,0.2416035703731357,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p001-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p005,0,RMSE,rmse,lower,0.23764755063810214,-0.23764755063810214,0.24545806951373206,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p005-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p001,0,RMSE,rmse,lower,0.23350649291605036,-0.23350649291605036,0.23477688348213874,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p001-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p005,0,RMSE,rmse,lower,0.23351058500178656,-0.23351058500178656,0.23619933599335108,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p005-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p001,0,RMSE,rmse,lower,0.23872630190199182,-0.23872630190199182,0.23512648270487807,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p001-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p001,0,RMSE,rmse,lower,0.23347937804489072,-0.23347937804489072,0.24206068964408525,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p001-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p005,0,RMSE,rmse,lower,0.23800428115225924,-0.23800428115225924,0.23509819987146183,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p005-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p005,0,RMSE,rmse,lower,0.23277299468998114,-0.23277299468998114,0.2357753957826797,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p005-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p01,0,RMSE,rmse,lower,0.2388627040189618,-0.2388627040189618,0.24858429278757524,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p01-best-head-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p01,0,RMSE,rmse,lower,0.23403326393093224,-0.23403326393093224,0.23267639873508839,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p01-greedy-heads-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p01,0,RMSE,rmse,lower,0.23798911368843165,-0.23798911368843165,0.24638280846527752,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p01-best-head-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p01,0,RMSE,rmse,lower,0.2329412795410917,-0.2329412795410917,0.23349850795747007,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p01-greedy-heads-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p001,0,RMSE,rmse,lower,0.23993622491293543,-0.23993622491293543,0.2477713353601464,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p001-best-head-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p001,0,RMSE,rmse,lower,0.23356126348868944,-0.23356126348868944,0.2502618550799444,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p001-greedy-heads-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p005,0,RMSE,rmse,lower,0.2360115531890879,-0.2360115531890879,0.23473422019733492,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p005-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p005,0,RMSE,rmse,lower,0.23283208982244355,-0.23283208982244355,0.23176779995051228,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p005-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p01,0,RMSE,rmse,lower,0.23637143045682502,-0.23637143045682502,0.2444339973181191,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p01-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p01,0,RMSE,rmse,lower,0.233185579288438,-0.233185579288438,0.23514442509167285,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p01-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p005,0,RMSE,rmse,lower,0.23936406325960766,-0.23936406325960766,0.23718997553210766,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p005-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p001,0,RMSE,rmse,lower,0.2389538449467578,-0.2389538449467578,0.24008690564440552,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p001-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p005,0,RMSE,rmse,lower,0.23354782526985796,-0.23354782526985796,0.23343612653897647,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p005-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p001,0,RMSE,rmse,lower,0.2336424824585913,-0.2336424824585913,0.23903173547895684,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p001-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p01,0,RMSE,rmse,lower,0.238226555972743,-0.238226555972743,0.2428780524434567,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p01-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p01,0,RMSE,rmse,lower,0.23345663231013183,-0.23345663231013183,0.23687685165587416,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p01-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p001,0,RMSE,rmse,lower,0.23805604517030857,-0.23805604517030857,0.24432422600723777,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p001-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p001,0,RMSE,rmse,lower,0.23337935525306813,-0.23337935525306813,0.2363605083000373,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p001-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p005,0,RMSE,rmse,lower,0.23774960884328808,-0.23774960884328808,0.23837226942825876,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p005-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p005,0,RMSE,rmse,lower,0.23309062467822936,-0.23309062467822936,0.2363639835864881,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p005-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p01,0,RMSE,rmse,lower,0.2385343005293775,-0.2385343005293775,0.24325157796763505,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p01-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p01,0,RMSE,rmse,lower,0.23315395335227065,-0.23315395335227065,0.2333720363863976,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p01-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p01,0,RMSE,rmse,lower,0.23735456374234598,-0.23735456374234598,0.24576564896936498,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p01-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p01,0,RMSE,rmse,lower,0.23287923675610403,-0.23287923675610403,0.23928534680182742,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p01-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p001,0,RMSE,rmse,lower,0.23903748624498683,-0.23903748624498683,0.23879927467488832,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p001-best-head-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p001,0,RMSE,rmse,lower,0.23308791568735607,-0.23308791568735607,0.23436618629413886,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p001-greedy-heads-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p005,0,RMSE,rmse,lower,0.23781616129168393,-0.23781616129168393,0.24546581719417238,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p005-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p005,0,RMSE,rmse,lower,0.23341984830196463,-0.23341984830196463,0.24027958886401746,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p005-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p005,0,RMSE,rmse,lower,0.238198961838987,-0.238198961838987,0.23998827851003937,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p005-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p005,0,RMSE,rmse,lower,0.2329018389418362,-0.2329018389418362,0.23679443729395636,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p005-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p001,0,RMSE,rmse,lower,0.23901917370395387,-0.23901917370395387,0.23709576833229662,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p001-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p001,0,RMSE,rmse,lower,0.23354370990991408,-0.23354370990991408,0.23867420719836574,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p001-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p001,0,RMSE,rmse,lower,0.23745188445279078,-0.23745188445279078,0.23531645200331283,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p001-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p001,0,RMSE,rmse,lower,0.2330749056995843,-0.2330749056995843,0.23692981324824752,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p001-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p005,0,RMSE,rmse,lower,0.23707190033649148,-0.23707190033649148,0.2397359546762736,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p005-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p005,0,RMSE,rmse,lower,0.23225748836355237,-0.23225748836355237,0.2373654401585729,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p005-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p01,0,RMSE,rmse,lower,0.2384660660716776,-0.2384660660716776,0.2608031155268875,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p01-best-head-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p01,0,RMSE,rmse,lower,0.23344479982230373,-0.23344479982230373,0.2326324214479151,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p01-greedy-heads-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p005,0,RMSE,rmse,lower,0.23788508328903052,-0.23788508328903052,0.2505198973693071,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p005-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p005,0,RMSE,rmse,lower,0.23340701642091277,-0.23340701642091277,0.24153220831875505,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p005-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p001,0,RMSE,rmse,lower,0.23792789399055542,-0.23792789399055542,0.23833634568163614,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p001-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p001,0,RMSE,rmse,lower,0.23351789154964117,-0.23351789154964117,0.2420072500053204,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p001-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p01,0,RMSE,rmse,lower,0.2386455487030765,-0.2386455487030765,0.27701433524933666,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p01-best-head-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p01,0,RMSE,rmse,lower,0.23375630575487258,-0.23375630575487258,0.23772689559358076,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p01-greedy-heads-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p001,0,RMSE,rmse,lower,0.23771184363825437,-0.23771184363825437,0.2596801100779992,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p001-best-head-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p001,0,RMSE,rmse,lower,0.2337060760344375,-0.2337060760344375,0.23617809041406845,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p001-greedy-heads-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p01,0,RMSE,rmse,lower,0.23901159112197984,-0.23901159112197984,0.24670214607411306,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p01-best-head-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p01,0,RMSE,rmse,lower,0.2334202712313093,-0.2334202712313093,0.2351349272835184,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p01-greedy-heads-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p005,0,RMSE,rmse,lower,0.23758917502158358,-0.23758917502158358,0.23986609072189577,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p005-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p005,0,RMSE,rmse,lower,0.23335307969400199,-0.23335307969400199,0.23233217223360508,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p005-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p01,0,RMSE,rmse,lower,0.23859870588230647,-0.23859870588230647,0.23721550415536108,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p01-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p01,0,RMSE,rmse,lower,0.23377952428751153,-0.23377952428751153,0.23516218393385976,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p01-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p001,0,RMSE,rmse,lower,0.23834036265829958,-0.23834036265829958,0.2355387485424496,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p001-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p001,0,RMSE,rmse,lower,0.2328865391175455,-0.2328865391175455,0.23281656140106766,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p001-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p001,0,RMSE,rmse,lower,0.2357278561917395,-0.2357278561917395,0.24177902137278479,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p001-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p001,0,RMSE,rmse,lower,0.2330191974784263,-0.2330191974784263,0.24024111265333,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p001-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p005,0,RMSE,rmse,lower,0.23782636654662812,-0.23782636654662812,0.23905601555002126,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p005-best-head-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p005,0,RMSE,rmse,lower,0.2332281025812143,-0.2332281025812143,0.23533248824072742,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p005-greedy-heads-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p01,0,RMSE,rmse,lower,0.2383424727161954,-0.2383424727161954,0.23929541934545473,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p01-best-head-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p01,0,RMSE,rmse,lower,0.23335380606300898,-0.23335380606300898,0.23440792329336166,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p01-greedy-heads-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p005,0,RMSE,rmse,lower,0.2361938000242818,-0.2361938000242818,0.23968299921179284,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p005-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p005,0,RMSE,rmse,lower,0.23281693742350693,-0.23281693742350693,0.23764734685399194,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p005-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p01,0,RMSE,rmse,lower,0.236445130001989,-0.236445130001989,0.2384320235467145,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p01-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p01,0,RMSE,rmse,lower,0.23288295595018185,-0.23288295595018185,0.2378111877355073,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p01-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p005,0,RMSE,rmse,lower,0.23844788931164226,-0.23844788931164226,0.239992780080121,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p005-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p005,0,RMSE,rmse,lower,0.23278091651666263,-0.23278091651666263,0.23692641701684622,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p005-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p01,0,RMSE,rmse,lower,0.23860736324692997,-0.23860736324692997,0.24541511515454548,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p01-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p01,0,RMSE,rmse,lower,0.23326554764868923,-0.23326554764868923,0.24138003765619687,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p01-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p001,0,RMSE,rmse,lower,0.237046057654127,-0.237046057654127,0.2530936074511429,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p001-best-head-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p001,0,RMSE,rmse,lower,0.23381808394805512,-0.23381808394805512,0.230425386066968,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p001-greedy-heads-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p005,0,RMSE,rmse,lower,0.2371704208371665,-0.2371704208371665,0.23451416334598613,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p005-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p005,0,RMSE,rmse,lower,0.23311429304688983,-0.23311429304688983,0.23653039079394922,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p005-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p01,0,RMSE,rmse,lower,0.23633714926648194,-0.23633714926648194,0.23801691214143877,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p01-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p01,0,RMSE,rmse,lower,0.2329376812231261,-0.2329376812231261,0.23722725849121748,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p01-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p001,0,RMSE,rmse,lower,0.2384361951499822,-0.2384361951499822,0.23890960422803306,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p001-best-head-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p001,0,RMSE,rmse,lower,0.2338562469701412,-0.2338562469701412,0.23357564989983745,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p001-greedy-heads-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p001,0,RMSE,rmse,lower,0.23738497878870984,-0.23738497878870984,0.2429803744292108,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p001-best-head-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p001,0,RMSE,rmse,lower,0.23371274685689286,-0.23371274685689286,0.23853136443639264,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p001-greedy-heads-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p005,0,RMSE,rmse,lower,0.23674080812018197,-0.23674080812018197,0.24022462096222602,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p005-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p005,0,RMSE,rmse,lower,0.2333506132153353,-0.2333506132153353,0.23993873237607374,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p005-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p01,0,RMSE,rmse,lower,0.24086258324668144,-0.24086258324668144,0.2704202668685561,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p01-best-head-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p01,0,RMSE,rmse,lower,0.23354353444721934,-0.23354353444721934,0.2390892602865205,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p01-greedy-heads-evaluation/0,False,888 +rescue_sberbank_combined_confirm,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001,0,RMSE,rmse,lower,0.235050071622046,-0.235050071622046,0.23184680318718037,mean,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/0,False,666 +rescue_sberbank_combined_confirm,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001,1,RMSE,rmse,lower,0.23618618194018487,-0.23618618194018487,0.23253236276211406,mean,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/1.toml,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/1,False,1184 +rescue_sberbank_combined_confirm,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001,2,RMSE,rmse,lower,0.23592642069691294,-0.23592642069691294,0.2326191697888986,mean,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/2.toml,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/2,False,888 +rescue_sberbank_combined_confirm,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001,2,RMSE,rmse,lower,0.23592642069691294,-0.23592642069691294,0.2326191697888986,mean,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/2.toml,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/2,False,888 +rescue_sberbank_combined_confirm,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001,0,RMSE,rmse,lower,0.235050071622046,-0.235050071622046,0.23184680318718037,mean,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/0,False,666 +rescue_sberbank_combined_confirm,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001,1,RMSE,rmse,lower,0.23618618194018487,-0.23618618194018487,0.23253236276211406,mean,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/1.toml,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/1,False,1184 +rescue_sberbank_combined_confirm,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001,0,RMSE,rmse,lower,0.23560357747729638,-0.23560357747729638,0.2389191702722735,best-head,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-best-head-evaluation/0,False,666 +rescue_sberbank_combined_confirm,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001,2,RMSE,rmse,lower,0.23847619665119374,-0.23847619665119374,0.2542706881867437,best-head,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/2.toml,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-best-head-evaluation/2,False,888 +rescue_sberbank_combined_confirm,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001,1,RMSE,rmse,lower,0.24254141605050564,-0.24254141605050564,0.24204913155172003,best-head,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/1.toml,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-best-head-evaluation/1,False,1184 +rescue_sberbank_combined_confirm,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001,0,RMSE,rmse,lower,0.2324412121462238,-0.2324412121462238,0.23820740696857753,greedy-heads,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined_confirm,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001,2,RMSE,rmse,lower,0.23411027740292373,-0.23411027740292373,0.24496736545125247,greedy-heads,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/2.toml,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-greedy-heads-evaluation/2,False,888 +rescue_sberbank_combined_confirm,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001,1,RMSE,rmse,lower,0.23397077829789442,-0.23397077829789442,0.23276530465893142,greedy-heads,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/1.toml,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-greedy-heads-evaluation/1,False,1184 +rescue_sberbank_combined_confirm,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001,0,RMSE,rmse,lower,0.23560357747729638,-0.23560357747729638,0.2389191702722735,best-head,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-best-head-evaluation/0,False,666 +rescue_sberbank_combined_confirm,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001,2,RMSE,rmse,lower,0.23847619665119374,-0.23847619665119374,0.2542706881867437,best-head,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/2.toml,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-best-head-evaluation/2,False,888 +rescue_sberbank_combined_confirm,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001,1,RMSE,rmse,lower,0.24254141605050564,-0.24254141605050564,0.24204913155172003,best-head,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/1.toml,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-best-head-evaluation/1,False,1184 +rescue_sberbank_combined_confirm,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001,0,RMSE,rmse,lower,0.2324412121462238,-0.2324412121462238,0.23820740696857753,greedy-heads,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined_confirm,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001,2,RMSE,rmse,lower,0.23411027740292373,-0.23411027740292373,0.24496736545125247,greedy-heads,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/2.toml,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-greedy-heads-evaluation/2,False,888 +rescue_sberbank_combined_confirm,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001,1,RMSE,rmse,lower,0.23397077829789442,-0.23397077829789442,0.23276530465893142,greedy-heads,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/1.toml,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-greedy-heads-evaluation/1,False,1184 diff --git a/tabm_integrated/paper/exp/final_integrated_summary.csv b/tabm_integrated/paper/exp/final_integrated_summary.csv new file mode 100644 index 000000000..c2e9ff847 --- /dev/null +++ b/tabm_integrated/paper/exp/final_integrated_summary.csv @@ -0,0 +1,31 @@ +dataset,task_type,variant,source_variant,selected_config,metric,direction,metric_direction,validation_metric,matched_validation_baseline,matched_validation_delta,matched_validation_percent_delta,test_metric,mean,std,test_mean,test_std,n_seeds,baseline_mean,baseline_std,matched_baseline_inference_mode,matched_baseline_mean,matched_baseline_std,absolute_delta,percent_delta,precision,inference_mode,selected_inference_mode,config_path,result_path,matched_baseline_result_path,status +sberbank-housing,regression,baseline_plr,baseline_plr,baseline_plr,RMSE,lower,lower,0.23552481981288087,0.23552481981288087,0.0,0.0,0.23441786078206484,0.23441786078206484,0.0035482159973780638,0.23441786078206484,0.0035482159973780638,3,0.23441786078206484,0.0035482159973780638,mean,0.23441786078206484,0.0035482159973780638,0.0,0.0,3 seeds,mean,mean,exp/integrated/final/sberbank-housing/baseline_plr-evaluation/0.toml;exp/integrated/final/sberbank-housing/baseline_plr-evaluation/1.toml;exp/integrated/final/sberbank-housing/baseline_plr-evaluation/2.toml,exp/integrated/final/sberbank-housing/baseline_plr-evaluation/0;exp/integrated/final/sberbank-housing/baseline_plr-evaluation/1;exp/integrated/final/sberbank-housing/baseline_plr-evaluation/2,exp/integrated/final/sberbank-housing/baseline_plr-evaluation/0;exp/integrated/final/sberbank-housing/baseline_plr-evaluation/1;exp/integrated/final/sberbank-housing/baseline_plr-evaluation/2,baseline +sberbank-housing,regression,best_rla_only,rla_rank2_noise0.001,rla_rank2_noise0.001,RMSE,lower,lower,0.23385412997377952,0.23375674885076472,-9.738112301480029e-05,-0.041659170694990484,0.2323679570932143,0.2323679570932143,0.002285510337847783,0.2323679570932143,0.002285510337847783,3,0.2341050589838414,0.002783845646101281,greedy-heads,0.2341050589838414,0.002783845646101281,0.0017371018906270908,0.7420180914360294,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/sberbank-housing/best_rla_only-evaluation/0.toml;exp/integrated/final/sberbank-housing/best_rla_only-evaluation/1.toml;exp/integrated/final/sberbank-housing/best_rla_only-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/2,exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/2,weak_win +sberbank-housing,regression,best_esam_only,esam_rho0.0025,esam_rho0.0025,RMSE,lower,lower,0.23381482399811443,0.23375674885076472,-5.8075147349706535e-05,-0.02484426551756285,0.23440607616465495,0.23440607616465495,0.004011724082624796,0.23440607616465495,0.004011724082624796,3,0.2341050589838414,0.002783845646101281,greedy-heads,0.2341050589838414,0.002783845646101281,-0.00030101718081354534,-0.12858209135682216,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/sberbank-housing/best_esam_only-evaluation/0.toml;exp/integrated/final/sberbank-housing/best_esam_only-evaluation/1.toml;exp/integrated/final/sberbank-housing/best_esam_only-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/2,exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/2,loss +sberbank-housing,regression,best_mfb_only,mfb_keep0.8,mfb_keep0.8,RMSE,lower,lower,0.23321906081613947,0.23375674885076472,0.0005376880346252555,0.23002032551732954,0.23509963234240183,0.23509963234240183,0.003956610803973734,0.23509963234240183,0.003956610803973734,3,0.2341050589838414,0.002783845646101281,greedy-heads,0.2341050589838414,0.002783845646101281,-0.00099457335856043,-0.4248406091169,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/0.toml;exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/1.toml;exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/2,exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/2,loss +sberbank-housing,regression,best_cf_fisd_only,cf_fisd_only,cf_fisd_only,RMSE,lower,lower,0.23407921163352594,0.23375674885076472,-0.00032246278276121254,-0.137948009777925,0.23309306252476805,0.23309306252476805,0.0005850816922811394,0.23309306252476805,0.0005850816922811394,3,0.2341050589838414,0.002783845646101281,greedy-heads,0.2341050589838414,0.002783845646101281,0.00101199645907335,0.4322830371398342,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/0.toml;exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/1.toml;exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/2,exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/2,weak_win +sberbank-housing,regression,best_combined,sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001,sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001,RMSE,lower,lower,0.23887373005966525,0.24017066037702445,0.0012969303173591962,0.5400036437936467,0.24507966333691242,0.24507966333691242,0.008112050773101143,0.24507966333691242,0.008112050773101143,3,0.2502002926581795,0.013600655537586393,best-head,0.2502002926581795,0.013600655537586393,0.005120629321267084,2.0466120430413817,3 seeds,best-head,best-head,exp/integrated/final/sberbank-housing/best_combined-evaluation/0.toml;exp/integrated/final/sberbank-housing/best_combined-evaluation/1.toml;exp/integrated/final/sberbank-housing/best_combined-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/0;exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/1;exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/2,exp/integrated/final/sberbank-housing/baseline_plr-best-head-evaluation/0;exp/integrated/final/sberbank-housing/baseline_plr-best-head-evaluation/1;exp/integrated/final/sberbank-housing/baseline_plr-best-head-evaluation/2,weak_win +ecom-offers,binclass,baseline_plr,baseline_plr,baseline_plr,AUROC,higher,higher,0.6214296509780997,0.6214296509780997,0.0,0.0,0.5900580480375381,0.5900580480375381,0.0009703638630149601,0.5900580480375381,0.0009703638630149601,3,0.5900580480375381,0.0009703638630149601,mean,0.5900580480375381,0.0009703638630149601,0.0,0.0,3 seeds,mean,mean,exp/integrated/final/ecom-offers/baseline_plr-evaluation/0.toml;exp/integrated/final/ecom-offers/baseline_plr-evaluation/1.toml;exp/integrated/final/ecom-offers/baseline_plr-evaluation/2.toml,exp/integrated/final/ecom-offers/baseline_plr-evaluation/0;exp/integrated/final/ecom-offers/baseline_plr-evaluation/1;exp/integrated/final/ecom-offers/baseline_plr-evaluation/2,exp/integrated/final/ecom-offers/baseline_plr-evaluation/0;exp/integrated/final/ecom-offers/baseline_plr-evaluation/1;exp/integrated/final/ecom-offers/baseline_plr-evaluation/2,baseline +ecom-offers,binclass,best_rla_only,rla_rank2_noise0.0001,rla_rank2_noise0.0001,AUROC,higher,higher,0.6348279404183594,0.635345130387201,-0.000517189968841536,-0.08140299564841912,0.5984469268269463,0.5984469268269463,0.002797349140724218,0.5984469268269463,0.002797349140724218,3,0.5996938524587336,0.0035785432553676923,best-head,0.5996938524587336,0.0035785432553676923,-0.001246925631787299,-0.20792703254751188,3 seeds,best-head,best-head,exp/integrated/final/ecom-offers/best_rla_only-evaluation/0.toml;exp/integrated/final/ecom-offers/best_rla_only-evaluation/1.toml;exp/integrated/final/ecom-offers/best_rla_only-evaluation/2.toml,exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/0;exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/1;exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/2,exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/0;exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/1;exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/2,loss +ecom-offers,binclass,best_esam_only,esam_rho0.001,esam_rho0.001,AUROC,higher,higher,0.6366563506545114,0.6365904008411623,6.594981334917538e-05,0.0103598504253335,0.6004196058355985,0.6004196058355985,0.003493353451214683,0.6004196058355985,0.003493353451214683,3,0.6003705999706029,0.0034459818868121214,greedy-heads,0.6003705999706029,0.0034459818868121214,4.900586499556603e-05,0.008162602398912538,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/ecom-offers/best_esam_only-evaluation/0.toml;exp/integrated/final/ecom-offers/best_esam_only-evaluation/1.toml;exp/integrated/final/ecom-offers/best_esam_only-evaluation/2.toml,exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/0;exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/1;exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/2,exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/2,weak_win +ecom-offers,binclass,best_mfb_only,mfb_keep0.7,mfb_keep0.7,AUROC,higher,higher,0.6539769427777475,0.635345130387201,0.018631812390546565,2.9325498063063304,0.6095164790194458,0.6095164790194458,0.0020894834660806928,0.6095164790194458,0.0020894834660806928,3,0.5996938524587336,0.0035785432553676923,best-head,0.5996938524587336,0.0035785432553676923,0.0098226265607122,1.637940179049629,3 seeds,best-head,best-head,exp/integrated/final/ecom-offers/best_mfb_only-evaluation/0.toml;exp/integrated/final/ecom-offers/best_mfb_only-evaluation/1.toml;exp/integrated/final/ecom-offers/best_mfb_only-evaluation/2.toml,exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/0;exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/1;exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/2,exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/0;exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/1;exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/2,clear_win +ecom-offers,binclass,best_cf_fisd_only,cf_fisd_lambda0.05,cf_fisd_lambda0.05,AUROC,higher,higher,0.636440295793426,0.6365904008411623,-0.00015010504773627087,-0.02357953364328597,0.6002048966489212,0.6002048966489212,0.0034582462660955543,0.6002048966489212,0.0034582462660955543,3,0.6003705999706029,0.0034459818868121214,greedy-heads,0.6003705999706029,0.0034459818868121214,-0.00016570332168175383,-0.02760017257505072,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/0.toml;exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/1.toml;exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/0;exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/1;exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/2,exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/2,loss +ecom-offers,binclass,best_combined,mfb_cf_fisd,mfb_cf_fisd,AUROC,higher,higher,0.6485201184226032,0.6365904008411623,0.011929717581440902,1.8740021159096183,0.6052109584242933,0.6052109584242933,0.003222479623952224,0.6052109584242933,0.003222479623952224,3,0.6003705999706029,0.0034459818868121214,greedy-heads,0.6003705999706029,0.0034459818868121214,0.004840358453690352,0.8062284285618516,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/ecom-offers/best_combined-evaluation/0.toml;exp/integrated/final/ecom-offers/best_combined-evaluation/1.toml;exp/integrated/final/ecom-offers/best_combined-evaluation/2.toml,exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/0;exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/1;exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/2,exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/2,clear_win +homesite-insurance,binclass,baseline_plr,baseline_plr,baseline_plr,AUROC,higher,higher,0.9584198249724393,0.9584198249724393,0.0,0.0,0.9624941853379376,0.9624941853379376,0.0001780794332642721,0.9624941853379376,0.0001780794332642721,3,0.9624941853379376,0.0001780794332642721,mean,0.9624941853379376,0.0001780794332642721,0.0,0.0,3 seeds,mean,mean,exp/integrated/final/homesite-insurance/baseline_plr-evaluation/0.toml;exp/integrated/final/homesite-insurance/baseline_plr-evaluation/1.toml;exp/integrated/final/homesite-insurance/baseline_plr-evaluation/2.toml,exp/integrated/final/homesite-insurance/baseline_plr-evaluation/0;exp/integrated/final/homesite-insurance/baseline_plr-evaluation/1;exp/integrated/final/homesite-insurance/baseline_plr-evaluation/2,exp/integrated/final/homesite-insurance/baseline_plr-evaluation/0;exp/integrated/final/homesite-insurance/baseline_plr-evaluation/1;exp/integrated/final/homesite-insurance/baseline_plr-evaluation/2,baseline +homesite-insurance,binclass,best_rla_only,rla_rank4_noise1e-05,rla_rank4_noise1e-05,AUROC,higher,higher,0.9589784960899528,0.9591396202785444,-0.00016112418859159927,-0.01679882523722741,0.9625640466562138,0.9625640466562138,0.0006085287596608143,0.9625640466562138,0.0006085287596608143,3,0.9626755517312511,0.00027672454564782147,greedy-heads,0.9626755517312511,0.00027672454564782147,-0.00011150507503732143,-0.011582830252289419,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/homesite-insurance/best_rla_only-evaluation/0.toml;exp/integrated/final/homesite-insurance/best_rla_only-evaluation/1.toml;exp/integrated/final/homesite-insurance/best_rla_only-evaluation/2.toml,exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/2,exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/2,loss +homesite-insurance,binclass,best_esam_only,esam_only,esam_only,AUROC,higher,higher,0.959080906574986,0.9591396202785444,-5.871370355836625e-05,-0.006121497049753315,0.9627318700697393,0.9627318700697393,0.0004492301463849077,0.9627318700697393,0.0004492301463849077,3,0.9626755517312511,0.00027672454564782147,greedy-heads,0.9626755517312511,0.00027672454564782147,5.6318338488181574e-05,0.005850188922623008,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/homesite-insurance/best_esam_only-evaluation/0.toml;exp/integrated/final/homesite-insurance/best_esam_only-evaluation/1.toml;exp/integrated/final/homesite-insurance/best_esam_only-evaluation/2.toml,exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/2,exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/2,weak_win +homesite-insurance,binclass,best_mfb_only,mfb_keep0.95,mfb_keep0.95,AUROC,higher,higher,0.959182264937648,0.9591396202785444,4.2644659103641125e-05,0.004446136746103415,0.962382329909275,0.962382329909275,0.0004577202193422504,0.962382329909275,0.0004577202193422504,3,0.9626755517312511,0.00027672454564782147,greedy-heads,0.9626755517312511,0.00027672454564782147,-0.00029322182197610847,-0.030459049411692845,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/0.toml;exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/1.toml;exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/2.toml,exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/2,exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/2,loss +homesite-insurance,binclass,best_cf_fisd_only,cf_fisd_only,cf_fisd_only,AUROC,higher,higher,0.959527988245724,0.9591396202785444,0.00038836796717967026,0.04049128604101289,0.9629076428862089,0.9629076428862089,0.00047402677133884024,0.9629076428862089,0.00047402677133884024,3,0.9626755517312511,0.00027672454564782147,greedy-heads,0.9626755517312511,0.00027672454564782147,0.00023209115495781418,0.024108969479948606,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/0.toml;exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/1.toml;exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/2,exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/2,weak_win +homesite-insurance,binclass,best_combined,mfb_cf_fisd,mfb_cf_fisd,AUROC,higher,higher,0.95963296602657,0.9591396202785444,0.0004933457480256198,0.05143628076612526,0.9628490210733115,0.9628490210733115,0.0004379684704485265,0.9628490210733115,0.0004379684704485265,3,0.9626755517312511,0.00027672454564782147,greedy-heads,0.9626755517312511,0.00027672454564782147,0.0001734693420604172,0.018019502183103577,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/homesite-insurance/best_combined-evaluation/0.toml;exp/integrated/final/homesite-insurance/best_combined-evaluation/1.toml;exp/integrated/final/homesite-insurance/best_combined-evaluation/2.toml,exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/2,exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/2,weak_win +cooking-time,regression,baseline_plr,baseline_plr,baseline_plr,RMSE,lower,lower,0.46270875116661525,0.46270875116661525,0.0,0.0,0.4803806470025843,0.4803806470025843,0.00012338540954176603,0.4803806470025843,0.00012338540954176603,3,0.4803806470025843,0.00012338540954176603,mean,0.4803806470025843,0.00012338540954176603,0.0,0.0,3 seeds,mean,mean,exp/integrated/final/cooking-time/baseline_plr-evaluation/0.toml;exp/integrated/final/cooking-time/baseline_plr-evaluation/1.toml;exp/integrated/final/cooking-time/baseline_plr-evaluation/2.toml,exp/integrated/final/cooking-time/baseline_plr-evaluation/0;exp/integrated/final/cooking-time/baseline_plr-evaluation/1;exp/integrated/final/cooking-time/baseline_plr-evaluation/2,exp/integrated/final/cooking-time/baseline_plr-evaluation/0;exp/integrated/final/cooking-time/baseline_plr-evaluation/1;exp/integrated/final/cooking-time/baseline_plr-evaluation/2,baseline +cooking-time,regression,best_rla_only,rla_rank4_noise0.001,rla_rank4_noise0.001,RMSE,lower,lower,0.46244170136784024,0.4624700316244145,2.8330256574282675e-05,0.006125857814996866,0.48014625604758127,0.48014625604758127,0.00016088993704992786,0.48014625604758127,0.00016088993704992786,3,0.48012538218537737,0.0002586745994143849,greedy-heads,0.48012538218537737,0.0002586745994143849,-2.08738622038962e-05,-0.004347585647083486,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/cooking-time/best_rla_only-evaluation/0.toml;exp/integrated/final/cooking-time/best_rla_only-evaluation/1.toml;exp/integrated/final/cooking-time/best_rla_only-evaluation/2.toml,exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/2,exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/2,loss +cooking-time,regression,best_esam_only,esam_rho0.0025,esam_rho0.0025,RMSE,lower,lower,0.46247755253655587,0.4624700316244145,-7.520912141345537e-06,-0.0016262485408899945,0.4800648691653293,0.4800648691653293,8.021206715574865e-05,0.4800648691653293,8.021206715574865e-05,3,0.48012538218537737,0.0002586745994143849,greedy-heads,0.48012538218537737,0.0002586745994143849,6.0513020048058586e-05,0.012603586957353233,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/cooking-time/best_esam_only-evaluation/0.toml;exp/integrated/final/cooking-time/best_esam_only-evaluation/1.toml;exp/integrated/final/cooking-time/best_esam_only-evaluation/2.toml,exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/2,exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/2,weak_win +cooking-time,regression,best_mfb_only,mfb_keep0.8,mfb_keep0.8,RMSE,lower,lower,0.4625031808631155,0.4624700316244145,-3.3149238700957984e-05,-0.007167867415002461,0.4799728414668436,0.4799728414668436,0.00019455249694504936,0.4799728414668436,0.00019455249694504936,3,0.48012538218537737,0.0002586745994143849,greedy-heads,0.48012538218537737,0.0002586745994143849,0.00015254071853376683,0.03177101736205868,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/cooking-time/best_mfb_only-evaluation/0.toml;exp/integrated/final/cooking-time/best_mfb_only-evaluation/1.toml;exp/integrated/final/cooking-time/best_mfb_only-evaluation/2.toml,exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/2,exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/2,weak_win +cooking-time,regression,best_cf_fisd_only,cf_fisd_lambda0.05,cf_fisd_lambda0.05,RMSE,lower,lower,0.46295431080753496,0.4624700316244145,-0.0004842791831204396,-0.10471579778249003,0.48033743363756964,0.48033743363756964,4.649528370692734e-05,0.48033743363756964,4.649528370692734e-05,3,0.48012538218537737,0.0002586745994143849,greedy-heads,0.48012538218537737,0.0002586745994143849,-0.0002120514521922745,-0.04416584918445345,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/0.toml;exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/1.toml;exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/2,exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/2,loss +cooking-time,regression,best_combined,rla_esam,rla_esam,RMSE,lower,lower,0.46244255150399494,0.4624700316244145,2.748012041958736e-05,0.005942032681137007,0.48002605538016,0.48002605538016,9.65539747406379e-05,0.48002605538016,9.65539747406379e-05,3,0.48012538218537737,0.0002586745994143849,greedy-heads,0.48012538218537737,0.0002586745994143849,9.932680521734794e-05,0.020687680531540333,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/cooking-time/best_combined-evaluation/0.toml;exp/integrated/final/cooking-time/best_combined-evaluation/1.toml;exp/integrated/final/cooking-time/best_combined-evaluation/2.toml,exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/2,exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/2,weak_win +delivery-eta,regression,baseline_plr,baseline_plr,baseline_plr,RMSE,lower,lower,0.5559497041253089,0.5559497041253089,0.0,0.0,0.5502258480608934,0.5502258480608934,0.0005822921162605647,0.5502258480608934,0.0005822921162605647,3,0.5502258480608934,0.0005822921162605647,mean,0.5502258480608934,0.0005822921162605647,0.0,0.0,3 seeds,mean,mean,exp/integrated/final/delivery-eta/baseline_plr-evaluation/0.toml;exp/integrated/final/delivery-eta/baseline_plr-evaluation/1.toml;exp/integrated/final/delivery-eta/baseline_plr-evaluation/2.toml,exp/integrated/final/delivery-eta/baseline_plr-evaluation/0;exp/integrated/final/delivery-eta/baseline_plr-evaluation/1;exp/integrated/final/delivery-eta/baseline_plr-evaluation/2,exp/integrated/final/delivery-eta/baseline_plr-evaluation/0;exp/integrated/final/delivery-eta/baseline_plr-evaluation/1;exp/integrated/final/delivery-eta/baseline_plr-evaluation/2,baseline +delivery-eta,regression,best_rla_only,deliv_rla_r4_n0p0001_lr0p5_fr0,deliv_rla_r4_n0p0001_lr0p5_fr0,RMSE,lower,lower,0.557867731128666,0.5579635797458202,9.584861715428161e-05,0.017178292747699653,0.5543336992032142,0.5543336992032142,0.002661909801131988,0.5543336992032142,0.002661909801131988,3,0.5566895883615489,0.000859735360596188,best-head,0.5566895883615489,0.000859735360596188,0.002355889158334734,0.42319619543605924,3 seeds,best-head,best-head,exp/integrated/final/delivery-eta/best_rla_only-evaluation/0.toml;exp/integrated/final/delivery-eta/best_rla_only-evaluation/1.toml;exp/integrated/final/delivery-eta/best_rla_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/0;exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/1;exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/2,exp/integrated/final/delivery-eta/baseline_plr-best-head-evaluation/0;exp/integrated/final/delivery-eta/baseline_plr-best-head-evaluation/1;exp/integrated/final/delivery-eta/baseline_plr-best-head-evaluation/2,clear_win +delivery-eta,regression,best_esam_only,deliv_esam_rho0p001_sf0p25_adapterf,deliv_esam_rho0p001_sf0p25_adapterf,RMSE,lower,lower,0.5559336909117792,0.5559497041253089,1.601321352961982e-05,0.002880334931522961,0.5512627378566571,0.5512627378566571,0.0012916491703925872,0.5512627378566571,0.0012916491703925872,3,0.5502258480608934,0.0005822921162605647,mean,0.5502258480608934,0.0005822921162605647,-0.0010368897957637735,-0.18844803446039146,3 seeds,mean,mean,exp/integrated/final/delivery-eta/best_esam_only-evaluation/0.toml;exp/integrated/final/delivery-eta/best_esam_only-evaluation/1.toml;exp/integrated/final/delivery-eta/best_esam_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_esam_only-evaluation/0;exp/integrated/final/delivery-eta/best_esam_only-evaluation/1;exp/integrated/final/delivery-eta/best_esam_only-evaluation/2,exp/integrated/final/delivery-eta/baseline_plr-evaluation/0;exp/integrated/final/delivery-eta/baseline_plr-evaluation/1;exp/integrated/final/delivery-eta/baseline_plr-evaluation/2,loss +delivery-eta,regression,best_mfb_only,deliv_mfb_k0p975_sf0p5_numerical_only,deliv_mfb_k0p975_sf0p5_numerical_only,RMSE,lower,lower,0.5560023662408545,0.5559497041253089,-5.2662115545665245e-05,-0.009472460396128821,0.5501995737833563,0.5501995737833563,0.0001818929992957081,0.5501995737833563,0.0001818929992957081,3,0.5502258480608934,0.0005822921162605647,mean,0.5502258480608934,0.0005822921162605647,2.6274277537097213e-05,0.004775180524450652,3 seeds,mean,mean,exp/integrated/final/delivery-eta/best_mfb_only-evaluation/0.toml;exp/integrated/final/delivery-eta/best_mfb_only-evaluation/1.toml;exp/integrated/final/delivery-eta/best_mfb_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_mfb_only-evaluation/0;exp/integrated/final/delivery-eta/best_mfb_only-evaluation/1;exp/integrated/final/delivery-eta/best_mfb_only-evaluation/2,exp/integrated/final/delivery-eta/baseline_plr-evaluation/0;exp/integrated/final/delivery-eta/baseline_plr-evaluation/1;exp/integrated/final/delivery-eta/baseline_plr-evaluation/2,weak_win +delivery-eta,regression,best_cf_fisd_only,deliv_cf_l0p005_consensus_raw,deliv_cf_l0p005_consensus_raw,RMSE,lower,lower,0.5553874207148172,0.5553874207148172,0.0,0.0,0.5511744515458483,0.5511744515458483,0.0007068650190236373,0.5511744515458483,0.0007068650190236373,3,0.5511744515458483,0.0007068650190236373,greedy-heads,0.5511744515458483,0.0007068650190236373,0.0,0.0,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/0.toml;exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/1.toml;exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/0;exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/1;exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/2,exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/2,tie +delivery-eta,regression,best_combined,deliv_comb_rla_cf,deliv_comb_rla_cf,RMSE,lower,lower,0.557867731128666,0.5579635797458202,9.584861715428161e-05,0.017178292747699653,0.5543336992032142,0.5543336992032142,0.002661909801131988,0.5543336992032142,0.002661909801131988,3,0.5566895883615489,0.000859735360596188,best-head,0.5566895883615489,0.000859735360596188,0.002355889158334734,0.42319619543605924,3 seeds,best-head,best-head,exp/integrated/final/delivery-eta/best_combined-evaluation/0.toml;exp/integrated/final/delivery-eta/best_combined-evaluation/1.toml;exp/integrated/final/delivery-eta/best_combined-evaluation/2.toml,exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/0;exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/1;exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/2,exp/integrated/final/delivery-eta/baseline_plr-best-head-evaluation/0;exp/integrated/final/delivery-eta/baseline_plr-best-head-evaluation/1;exp/integrated/final/delivery-eta/baseline_plr-best-head-evaluation/2,clear_win diff --git a/tabm_integrated/paper/exp/integrated/baseline_fidelity/cooking-time/baseline_plr-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/baseline_fidelity/cooking-time/baseline_plr-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/baseline_fidelity/cooking-time/baseline_plr-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/baseline_fidelity/cooking-time/baseline_plr-best-head-evaluation/0/report.json new file mode 100644 index 000000000..c9ff6ead7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/baseline_fidelity/cooking-time/baseline_plr-best-head-evaluation/0/report.json @@ -0,0 +1,147 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44650602179756516, + "mae": 0.34757497906684875, + "r2": 0.5420529246330261, + "score": -0.44650602179756516 + }, + "val": { + "rmse": 0.4652274815891859, + "mae": 0.3617282807826996, + "r2": 0.503640353679657, + "score": -0.4652274815891859 + }, + "test": { + "rmse": 0.4820660657381116, + "mae": 0.3761262893676758, + "r2": 0.48187607526779175, + "score": -0.4820660657381116 + } + }, + "time": "0:09:50.419637", + "diversity": { + "train": { + "mean_centered_corr": -0.020426616072654724, + "mean_pairwise_disagreement": NaN, + "member_std": 0.051189813762903214 + }, + "val": { + "mean_centered_corr": -0.02039431221783161, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05273108929395676 + }, + "test": { + "mean_centered_corr": -0.020482471212744713, + "mean_pairwise_disagreement": NaN, + "member_std": 0.053407639265060425 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 27 + ], + "head_selection_time": "0:00:00.027737" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/baseline_fidelity/cooking-time/baseline_plr-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/baseline_fidelity/cooking-time/baseline_plr-evaluation/0.toml new file mode 100644 index 000000000..78c613dfb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/baseline_fidelity/cooking-time/baseline_plr-evaluation/0.toml @@ -0,0 +1,33 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/baseline_fidelity/cooking-time/baseline_plr-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/baseline_fidelity/cooking-time/baseline_plr-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/baseline_fidelity/cooking-time/baseline_plr-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/baseline_fidelity/cooking-time/baseline_plr-evaluation/0/report.json new file mode 100644 index 000000000..21d650115 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/baseline_fidelity/cooking-time/baseline_plr-evaluation/0/report.json @@ -0,0 +1,143 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44389690465868414, + "mae": 0.34541288018226624, + "r2": 0.5473892688751221, + "score": -0.44389690465868414 + }, + "val": { + "rmse": 0.4627596029926704, + "mae": 0.35959550738334656, + "r2": 0.5088924169540405, + "score": -0.4627596029926704 + }, + "test": { + "rmse": 0.4802386655626689, + "mae": 0.37469902634620667, + "r2": 0.4857967495918274, + "score": -0.4802386655626689 + } + }, + "time": "0:09:50.419637", + "diversity": { + "train": { + "mean_centered_corr": -0.020426616072654724, + "mean_pairwise_disagreement": NaN, + "member_std": 0.051189813762903214 + }, + "val": { + "mean_centered_corr": -0.02039431221783161, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05273108929395676 + }, + "test": { + "mean_centered_corr": -0.020482471212744713, + "mean_pairwise_disagreement": NaN, + "member_std": 0.053407639265060425 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/baseline_fidelity/cooking-time/baseline_plr-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/baseline_fidelity/cooking-time/baseline_plr-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/baseline_fidelity/cooking-time/baseline_plr-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/baseline_fidelity/cooking-time/baseline_plr-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..9addb5172 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/baseline_fidelity/cooking-time/baseline_plr-greedy-heads-evaluation/0/report.json @@ -0,0 +1,157 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44394242184625543, + "mae": 0.3455282151699066, + "r2": 0.547296404838562, + "score": -0.44394242184625543 + }, + "val": { + "rmse": 0.4625169879783321, + "mae": 0.3595449924468994, + "r2": 0.5094072818756104, + "score": -0.4625169879783321 + }, + "test": { + "rmse": 0.47990786370986477, + "mae": 0.3745970129966736, + "r2": 0.4865049123764038, + "score": -0.47990786370986477 + } + }, + "time": "0:09:50.419637", + "diversity": { + "train": { + "mean_centered_corr": -0.020426616072654724, + "mean_pairwise_disagreement": NaN, + "member_std": 0.051189813762903214 + }, + "val": { + "mean_centered_corr": -0.02039431221783161, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05273108929395676 + }, + "test": { + "mean_centered_corr": -0.020482471212744713, + "mean_pairwise_disagreement": NaN, + "member_std": 0.053407639265060425 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 27, + 17, + 5, + 3, + 14, + 4, + 22, + 6, + 24, + 10, + 16 + ], + "head_selection_time": "0:00:00.401807" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/baseline_fidelity/delivery-eta/baseline_plr-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/baseline_fidelity/delivery-eta/baseline_plr-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/baseline_fidelity/delivery-eta/baseline_plr-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/baseline_fidelity/delivery-eta/baseline_plr-best-head-evaluation/0/report.json new file mode 100644 index 000000000..4fd0f8dbc --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/baseline_fidelity/delivery-eta/baseline_plr-best-head-evaluation/0/report.json @@ -0,0 +1,168 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5112224822345067, + "mae": 0.40589213371276855, + "r2": 0.5101465582847595, + "score": -0.5112224822345067 + }, + "val": { + "rmse": 0.5583881453279098, + "mae": 0.4439501166343689, + "r2": 0.412036657333374, + "score": -0.5583881453279098 + }, + "test": { + "rmse": 0.5557654209580059, + "mae": 0.4505227506160736, + "r2": 0.42443811893463135, + "score": -0.5557654209580059 + } + }, + "time": "0:07:15.524601", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023608" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/baseline_fidelity/delivery-eta/baseline_plr-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/baseline_fidelity/delivery-eta/baseline_plr-evaluation/0.toml new file mode 100644 index 000000000..d07370fc8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/baseline_fidelity/delivery-eta/baseline_plr-evaluation/0.toml @@ -0,0 +1,33 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/baseline_fidelity/delivery-eta/baseline_plr-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/baseline_fidelity/delivery-eta/baseline_plr-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/baseline_fidelity/delivery-eta/baseline_plr-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/baseline_fidelity/delivery-eta/baseline_plr-evaluation/0/report.json new file mode 100644 index 000000000..95e980416 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/baseline_fidelity/delivery-eta/baseline_plr-evaluation/0/report.json @@ -0,0 +1,164 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5040057771555991, + "mae": 0.39488768577575684, + "r2": 0.5238790512084961, + "score": -0.5040057771555991 + }, + "val": { + "rmse": 0.5558749906446844, + "mae": 0.4398687481880188, + "r2": 0.4173172116279602, + "score": -0.5558749906446844 + }, + "test": { + "rmse": 0.5496162918403876, + "mae": 0.44109272956848145, + "r2": 0.4371039867401123, + "score": -0.5496162918403876 + } + }, + "time": "0:07:15.524601", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/baseline_fidelity/delivery-eta/baseline_plr-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/baseline_fidelity/delivery-eta/baseline_plr-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/baseline_fidelity/delivery-eta/baseline_plr-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/baseline_fidelity/delivery-eta/baseline_plr-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..b9623297c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/baseline_fidelity/delivery-eta/baseline_plr-greedy-heads-evaluation/0/report.json @@ -0,0 +1,181 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5042286809765437, + "mae": 0.39605751633644104, + "r2": 0.5234577655792236, + "score": -0.5042286809765437 + }, + "val": { + "rmse": 0.5553348954063525, + "mae": 0.44016024470329285, + "r2": 0.4184489846229553, + "score": -0.5553348954063525 + }, + "test": { + "rmse": 0.550360417400049, + "mae": 0.44305530190467834, + "r2": 0.43557870388031006, + "score": -0.550360417400049 + } + }, + "time": "0:07:15.524601", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13, + 6, + 10, + 14, + 26, + 27, + 18, + 25, + 6, + 15, + 8, + 17, + 10, + 26 + ], + "head_selection_time": "0:00:00.441754" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/baseline_fidelity/ecom-offers/baseline_plr-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/baseline_fidelity/ecom-offers/baseline_plr-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/baseline_fidelity/ecom-offers/baseline_plr-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/baseline_fidelity/ecom-offers/baseline_plr-best-head-evaluation/0/report.json new file mode 100644 index 000000000..588061229 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/baseline_fidelity/ecom-offers/baseline_plr-best-head-evaluation/0/report.json @@ -0,0 +1,219 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7835229427379597, + "recall": 0.9817888071846897, + "f1-score": 0.871522047464133, + "support": 84179.0 + }, + "1": { + "precision": 0.602952602952603, + "recall": 0.09252046737143312, + "f1-score": 0.1604244909209937, + "support": 25162.0 + }, + "accuracy": 0.7771467244674916, + "macro avg": { + "precision": 0.6932377728452814, + "recall": 0.5371546372780615, + "f1-score": 0.5159732691925634, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7419693545168977, + "recall": 0.7771467244674916, + "f1-score": 0.707881357167369, + "support": 109341.0 + }, + "cross-entropy": 0.49822654924283316, + "roc-auc": 0.6867900811541505, + "score": 0.6867900811541505 + }, + "val": { + "0": { + "precision": 0.6877843221759325, + "recall": 0.9255703054517335, + "f1-score": 0.7891541441169199, + "support": 15518.0 + }, + "1": { + "precision": 0.6580817051509769, + "recall": 0.2542605512981814, + "f1-score": 0.36680141902483293, + "support": 8743.0 + }, + "accuracy": 0.6836486542187049, + "macro avg": { + "precision": 0.6729330136634547, + "recall": 0.5899154283749575, + "f1-score": 0.5779777815708764, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6770803124216278, + "recall": 0.6836486542187049, + "f1-score": 0.6369497883409784, + "support": 24261.0 + }, + "cross-entropy": 0.6337455659020594, + "roc-auc": 0.6404461738890127, + "score": 0.6404461738890127 + }, + "test": { + "0": { + "precision": 0.6663787430075683, + "recall": 0.9573927431745657, + "f1-score": 0.785807828491051, + "support": 16922.0 + }, + "1": { + "precision": 0.6635557629491368, + "recall": 0.14916605475715933, + "f1-score": 0.24357656731757452, + "support": 9533.0 + }, + "accuracy": 0.6661500661500661, + "macro avg": { + "precision": 0.6649672529783526, + "recall": 0.5532793989658625, + "f1-score": 0.5146921979043128, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6653614884660062, + "recall": 0.6661500661500661, + "f1-score": 0.5904160079366473, + "support": 26455.0 + }, + "cross-entropy": 0.6685406788276543, + "roc-auc": 0.6034781946000056, + "score": 0.6034781946000056 + } + }, + "time": "0:01:00.962097", + "diversity": { + "train": { + "mean_centered_corr": -0.013087093830108643, + "mean_pairwise_disagreement": 0.025822423398123243, + "member_std": 0.02709772437810898 + }, + "val": { + "mean_centered_corr": -0.004288136959075928, + "mean_pairwise_disagreement": 0.08692423855623854, + "member_std": 0.060995232313871384 + }, + "test": { + "mean_centered_corr": 0.0036114430986344814, + "mean_pairwise_disagreement": 0.05469726437468373, + "member_std": 0.05942297354340553 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 14 + ], + "head_selection_time": "0:00:04.742166" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/baseline_fidelity/ecom-offers/baseline_plr-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/baseline_fidelity/ecom-offers/baseline_plr-evaluation/0.toml new file mode 100644 index 000000000..bf2cd1131 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/baseline_fidelity/ecom-offers/baseline_plr-evaluation/0.toml @@ -0,0 +1,33 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/baseline_fidelity/ecom-offers/baseline_plr-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/baseline_fidelity/ecom-offers/baseline_plr-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/baseline_fidelity/ecom-offers/baseline_plr-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/baseline_fidelity/ecom-offers/baseline_plr-evaluation/0/report.json new file mode 100644 index 000000000..329fc00d3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/baseline_fidelity/ecom-offers/baseline_plr-evaluation/0/report.json @@ -0,0 +1,215 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7862557522292872, + "recall": 0.9783200085532021, + "f1-score": 0.8718353174078054, + "support": 84179.0 + }, + "1": { + "precision": 0.6031746031746031, + "recall": 0.11024560845719736, + "f1-score": 0.18641846712140048, + "support": 25162.0 + }, + "accuracy": 0.7785551622904492, + "macro avg": { + "precision": 0.6947151777019451, + "recall": 0.5442828085051997, + "f1-score": 0.529126892264603, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7441243662668947, + "recall": 0.7785551622904492, + "f1-score": 0.7141043767093801, + "support": 109341.0 + }, + "cross-entropy": 0.4922807370219411, + "roc-auc": 0.6989988975549913, + "score": 0.6989988975549913 + }, + "val": { + "0": { + "precision": 0.6750500272876114, + "recall": 0.9565021265627014, + "f1-score": 0.7915000266624007, + "support": 15518.0 + }, + "1": { + "precision": 0.7030356357237132, + "recall": 0.1827747912615807, + "f1-score": 0.29012345679012347, + "support": 8743.0 + }, + "accuracy": 0.6776719838423808, + "macro avg": { + "precision": 0.6890428315056623, + "recall": 0.569638458912141, + "f1-score": 0.5408117417262621, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6851352741676592, + "recall": 0.6776719838423808, + "f1-score": 0.6108176413364323, + "support": 24261.0 + }, + "cross-entropy": 0.6872792555776136, + "roc-auc": 0.6227054517511603, + "score": 0.6227054517511603 + }, + "test": { + "0": { + "precision": 0.656363924552227, + "recall": 0.9788441082614349, + "f1-score": 0.7858057782627259, + "support": 16922.0 + }, + "1": { + "precision": 0.7063166529942576, + "recall": 0.09031784328123361, + "f1-score": 0.16015625, + "support": 9533.0 + }, + "accuracy": 0.6586656586656586, + "macro avg": { + "precision": 0.6813402887732423, + "recall": 0.5345809757713342, + "f1-score": 0.47298101413136295, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6743642783695727, + "recall": 0.6586656586656586, + "f1-score": 0.5603543719906198, + "support": 26455.0 + }, + "cross-entropy": 0.756433532699498, + "roc-auc": 0.591151981311678, + "score": 0.591151981311678 + } + }, + "time": "0:01:00.962097", + "diversity": { + "train": { + "mean_centered_corr": -0.013087093830108643, + "mean_pairwise_disagreement": 0.025822423398123243, + "member_std": 0.02709772437810898 + }, + "val": { + "mean_centered_corr": -0.004288136959075928, + "mean_pairwise_disagreement": 0.08692423855623854, + "member_std": 0.060995232313871384 + }, + "test": { + "mean_centered_corr": 0.0036114430986344814, + "mean_pairwise_disagreement": 0.05469726437468373, + "member_std": 0.05942297354340553 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/baseline_fidelity/ecom-offers/baseline_plr-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/baseline_fidelity/ecom-offers/baseline_plr-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/baseline_fidelity/ecom-offers/baseline_plr-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/baseline_fidelity/ecom-offers/baseline_plr-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..d0ce06b82 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/baseline_fidelity/ecom-offers/baseline_plr-greedy-heads-evaluation/0/report.json @@ -0,0 +1,227 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7829606930730526, + "recall": 0.9834281709214887, + "f1-score": 0.8718189046448073, + "support": 84179.0 + }, + "1": { + "precision": 0.6134663341645885, + "recall": 0.08798982592798664, + "f1-score": 0.15390497375829829, + "support": 25162.0 + }, + "accuracy": 0.7773662212710694, + "macro avg": { + "precision": 0.6982135136188206, + "recall": 0.5357089984247376, + "f1-score": 0.5128619392015528, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7439559550621073, + "recall": 0.7773662212710694, + "f1-score": 0.7066096022882681, + "support": 109341.0 + }, + "cross-entropy": 0.4977984361309486, + "roc-auc": 0.6854164410903827, + "score": 0.6854164410903827 + }, + "val": { + "0": { + "precision": 0.6785484611851171, + "recall": 0.9519267946900374, + "f1-score": 0.7923192447972538, + "support": 15518.0 + }, + "1": { + "precision": 0.7005218787635488, + "recall": 0.19958824202218917, + "f1-score": 0.31066405554566495, + "support": 8743.0 + }, + "accuracy": 0.6808045834879024, + "macro avg": { + "precision": 0.689535169974333, + "recall": 0.5757575183561132, + "f1-score": 0.5514916501714594, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6864670791270086, + "recall": 0.6808045834879024, + "f1-score": 0.6187439049668, + "support": 24261.0 + }, + "cross-entropy": 0.6384330519984234, + "roc-auc": 0.6413971749638401, + "score": 0.6413971749638401 + }, + "test": { + "0": { + "precision": 0.659637469489016, + "recall": 0.9741756293582319, + "f1-score": 0.7866294467110443, + "support": 16922.0 + }, + "1": { + "precision": 0.7015027322404371, + "recall": 0.1077310395468373, + "f1-score": 0.18677821223970173, + "support": 9533.0 + }, + "accuracy": 0.661954261954262, + "macro avg": { + "precision": 0.6805701008647266, + "recall": 0.5409533344525346, + "f1-score": 0.486703829475373, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6747235231578611, + "recall": 0.661954261954262, + "f1-score": 0.5704743978274568, + "support": 26455.0 + }, + "cross-entropy": 0.6791357747346614, + "roc-auc": 0.6040305589800323, + "score": 0.6040305589800323 + } + }, + "time": "0:01:00.962097", + "diversity": { + "train": { + "mean_centered_corr": -0.013087093830108643, + "mean_pairwise_disagreement": 0.025822423398123243, + "member_std": 0.02709772437810898 + }, + "val": { + "mean_centered_corr": -0.004288136959075928, + "mean_pairwise_disagreement": 0.08692423855623854, + "member_std": 0.060995232313871384 + }, + "test": { + "mean_centered_corr": 0.0036114430986344814, + "mean_pairwise_disagreement": 0.05469726437468373, + "member_std": 0.05942297354340553 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 14, + 0, + 25, + 25, + 0, + 25, + 0, + 25, + 0 + ], + "head_selection_time": "0:00:48.379804" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/baseline_fidelity/homesite-insurance/baseline_plr-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/baseline_fidelity/homesite-insurance/baseline_plr-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/baseline_fidelity/homesite-insurance/baseline_plr-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/baseline_fidelity/homesite-insurance/baseline_plr-best-head-evaluation/0/report.json new file mode 100644 index 000000000..2d7cfcb35 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/baseline_fidelity/homesite-insurance/baseline_plr-best-head-evaluation/0/report.json @@ -0,0 +1,262 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9509438009438009, + "recall": 0.9745069269279171, + "f1-score": 0.9625811844722908, + "support": 181971.0 + }, + "1": { + "precision": 0.8774048625792812, + "recall": 0.7839854542019883, + "f1-score": 0.8280686877252491, + "support": 42349.0 + }, + "accuracy": 0.9385386947218259, + "macro avg": { + "precision": 0.9141743317615411, + "recall": 0.8792461905649527, + "f1-score": 0.89532493609877, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9370605069851746, + "recall": 0.9385386947218259, + "f1-score": 0.9371867937592895, + "support": 224320.0 + }, + "cross-entropy": 0.14097183098455576, + "roc-auc": 0.9772540623630491, + "score": 0.9772540623630491 + }, + "val": { + "0": { + "precision": 0.9410365039753932, + "recall": 0.9685801326085658, + "f1-score": 0.9546096785588132, + "support": 16741.0 + }, + "1": { + "precision": 0.8190574475404196, + "recall": 0.7009125699146306, + "f1-score": 0.7553934010152284, + "support": 3397.0 + }, + "accuracy": 0.923428344423478, + "macro avg": { + "precision": 0.8800469757579064, + "recall": 0.8347463512615982, + "f1-score": 0.8550015397870208, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9204603367934683, + "recall": 0.923428344423478, + "f1-score": 0.9210046683882125, + "support": 20138.0 + }, + "cross-entropy": 0.181709302619563, + "roc-auc": 0.9554763470552776, + "score": 0.9554763470552776 + }, + "test": { + "0": { + "precision": 0.9398939744642724, + "recall": 0.9574807940975127, + "f1-score": 0.9486058779201205, + "support": 13147.0 + }, + "1": { + "precision": 0.8073742246726395, + "recall": 0.7442820838627701, + "f1-score": 0.7745454545454545, + "support": 3148.0 + }, + "accuracy": 0.9162933415158024, + "macro avg": { + "precision": 0.873634099568456, + "recall": 0.8508814389801413, + "f1-score": 0.8615756662327876, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9142927365174138, + "recall": 0.9162933415158024, + "f1-score": 0.914979476399197, + "support": 16295.0 + }, + "cross-entropy": 0.18955508005758365, + "roc-auc": 0.959416932798502, + "score": 0.959416932798502 + } + }, + "time": "0:02:48.131677", + "diversity": { + "train": { + "mean_centered_corr": 0.05291927233338356, + "mean_pairwise_disagreement": 0.03400679940235148, + "member_std": 0.03158120810985565 + }, + "val": { + "mean_centered_corr": 0.06288693100214005, + "mean_pairwise_disagreement": 0.03244768356405319, + "member_std": 0.0349886491894722 + }, + "test": { + "mean_centered_corr": 0.07999585568904877, + "mean_pairwise_disagreement": 0.041502439893495925, + "member_std": 0.0369756743311882 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31 + ], + "head_selection_time": "0:00:00.912933" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/baseline_fidelity/homesite-insurance/baseline_plr-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/baseline_fidelity/homesite-insurance/baseline_plr-evaluation/0.toml new file mode 100644 index 000000000..d85b237a3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/baseline_fidelity/homesite-insurance/baseline_plr-evaluation/0.toml @@ -0,0 +1,34 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/baseline_fidelity/homesite-insurance/baseline_plr-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/baseline_fidelity/homesite-insurance/baseline_plr-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/baseline_fidelity/homesite-insurance/baseline_plr-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/baseline_fidelity/homesite-insurance/baseline_plr-evaluation/0/report.json new file mode 100644 index 000000000..3e79b9f86 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/baseline_fidelity/homesite-insurance/baseline_plr-evaluation/0/report.json @@ -0,0 +1,258 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9482836172363248, + "recall": 0.984772298882789, + "f1-score": 0.966183574879227, + "support": 181971.0 + }, + "1": { + "precision": 0.9216057939853453, + "recall": 0.769227136414083, + "f1-score": 0.8385502471169687, + "support": 42349.0 + }, + "accuracy": 0.9440798858773182, + "macro avg": { + "precision": 0.9349447056108351, + "recall": 0.8769997176484361, + "f1-score": 0.9023669109980978, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.943247155320955, + "recall": 0.9440798858773182, + "f1-score": 0.9420878910462924, + "support": 224320.0 + }, + "cross-entropy": 0.13271224543526663, + "roc-auc": 0.9813876825616359, + "score": 0.9813876825616359 + }, + "val": { + "0": { + "precision": 0.9351403678606002, + "recall": 0.9809449853652709, + "f1-score": 0.9574951897848522, + "support": 16741.0 + }, + "1": { + "precision": 0.8762126503686457, + "recall": 0.6647041507212246, + "f1-score": 0.7559424171409441, + "support": 3397.0 + }, + "accuracy": 0.9275995630151952, + "macro avg": { + "precision": 0.9056765091146229, + "recall": 0.8228245680432478, + "f1-score": 0.8567188034628981, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9252000830100605, + "recall": 0.9275995630151952, + "f1-score": 0.9234960454472141, + "support": 20138.0 + }, + "cross-entropy": 0.17351272311805546, + "roc-auc": 0.9583121450834431, + "score": 0.9583121450834431 + }, + "test": { + "0": { + "precision": 0.9321849105974706, + "recall": 0.975507720392485, + "f1-score": 0.9533543950938487, + "support": 13147.0 + }, + "1": { + "precision": 0.8730784391013008, + "recall": 0.7036213468869124, + "f1-score": 0.7792436235708003, + "support": 3148.0 + }, + "accuracy": 0.9229825099723842, + "macro avg": { + "precision": 0.9026316748493857, + "recall": 0.8395645336396986, + "f1-score": 0.8662990093323245, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9207662439960627, + "recall": 0.9229825099723842, + "f1-score": 0.9197182669100772, + "support": 16295.0 + }, + "cross-entropy": 0.17696141048282132, + "roc-auc": 0.9626260391126089, + "score": 0.9626260391126089 + } + }, + "time": "0:02:48.131677", + "diversity": { + "train": { + "mean_centered_corr": 0.05291927233338356, + "mean_pairwise_disagreement": 0.03400679940235148, + "member_std": 0.03158120810985565 + }, + "val": { + "mean_centered_corr": 0.06288693100214005, + "mean_pairwise_disagreement": 0.03244768356405319, + "member_std": 0.0349886491894722 + }, + "test": { + "mean_centered_corr": 0.07999585568904877, + "mean_pairwise_disagreement": 0.041502439893495925, + "member_std": 0.0369756743311882 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/baseline_fidelity/homesite-insurance/baseline_plr-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/baseline_fidelity/homesite-insurance/baseline_plr-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/baseline_fidelity/homesite-insurance/baseline_plr-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/baseline_fidelity/homesite-insurance/baseline_plr-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..d5d5b7b36 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/baseline_fidelity/homesite-insurance/baseline_plr-greedy-heads-evaluation/0/report.json @@ -0,0 +1,275 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9453926712602988, + "recall": 0.9862120887394146, + "f1-score": 0.9653710741556594, + "support": 181971.0 + }, + "1": { + "precision": 0.9272584947234142, + "recall": 0.7552244444969185, + "f1-score": 0.8324462201168646, + "support": 42349.0 + }, + "accuracy": 0.9426043152639088, + "macro avg": { + "precision": 0.9363255829918564, + "recall": 0.8707182666181665, + "f1-score": 0.898908647136262, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9419691502092978, + "recall": 0.9426043152639088, + "f1-score": 0.9402764118710262, + "support": 224320.0 + }, + "cross-entropy": 0.13432360872538882, + "roc-auc": 0.9807020207732833, + "score": 0.9807020207732833 + }, + "val": { + "0": { + "precision": 0.9332728098048116, + "recall": 0.9824980586583836, + "f1-score": 0.9572530190600902, + "support": 16741.0 + }, + "1": { + "precision": 0.8834526650755767, + "recall": 0.6538121872240212, + "f1-score": 0.7514802909829132, + "support": 3397.0 + }, + "accuracy": 0.927053332009137, + "macro avg": { + "precision": 0.9083627374401941, + "recall": 0.8181551229412024, + "f1-score": 0.8543666550215017, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9248688455757318, + "recall": 0.927053332009137, + "f1-score": 0.9225420270411127, + "support": 20138.0 + }, + "cross-entropy": 0.17301902153958362, + "roc-auc": 0.9589474980445032, + "score": 0.9589474980445032 + }, + "test": { + "0": { + "precision": 0.9310144506571781, + "recall": 0.9752034684718948, + "f1-score": 0.952596775391931, + "support": 13147.0 + }, + "1": { + "precision": 0.8708399366085579, + "recall": 0.6982210927573063, + "f1-score": 0.7750352609308886, + "support": 3148.0 + }, + "accuracy": 0.921693771095428, + "macro avg": { + "precision": 0.900927193632868, + "recall": 0.8367122806146006, + "f1-score": 0.8638160181614098, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9193894509502093, + "recall": 0.921693771095428, + "f1-score": 0.9182940047553332, + "support": 16295.0 + }, + "cross-entropy": 0.17739591453615167, + "roc-auc": 0.9625170670540112, + "score": 0.9625170670540112 + } + }, + "time": "0:02:48.131677", + "diversity": { + "train": { + "mean_centered_corr": 0.05291927233338356, + "mean_pairwise_disagreement": 0.03400679940235148, + "member_std": 0.03158120810985565 + }, + "val": { + "mean_centered_corr": 0.06288693100214005, + "mean_pairwise_disagreement": 0.03244768356405319, + "member_std": 0.0349886491894722 + }, + "test": { + "mean_centered_corr": 0.07999585568904877, + "mean_pairwise_disagreement": 0.041502439893495925, + "member_std": 0.0369756743311882 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31, + 6, + 9, + 15, + 28, + 3, + 23, + 26, + 2, + 20, + 13, + 6, + 15, + 5 + ], + "head_selection_time": "0:00:13.989889" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/baseline_fidelity/sberbank-housing/baseline_plr-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/baseline_fidelity/sberbank-housing/baseline_plr-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/baseline_fidelity/sberbank-housing/baseline_plr-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/baseline_fidelity/sberbank-housing/baseline_plr-best-head-evaluation/0/report.json new file mode 100644 index 000000000..ff85a29bb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/baseline_fidelity/sberbank-housing/baseline_plr-best-head-evaluation/0/report.json @@ -0,0 +1,190 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22658863410177663, + "mae": 0.12739618122577667, + "r2": 0.6719876527786255, + "score": -0.22658863410177663 + }, + "val": { + "rmse": 0.23955454428788905, + "mae": 0.13915589451789856, + "r2": 0.6166812181472778, + "score": -0.23955454428788905 + }, + "test": { + "rmse": 0.26588264760298186, + "mae": 0.1863766759634018, + "r2": 0.5169194340705872, + "score": -0.26588264760298186 + } + }, + "time": "0:00:48.779291", + "diversity": { + "train": { + "mean_centered_corr": -0.018623020499944687, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05047355964779854 + }, + "val": { + "mean_centered_corr": -0.01870153285562992, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06864207983016968 + }, + "test": { + "mean_centered_corr": -0.019026711583137512, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09529124945402145 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015297" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/baseline_fidelity/sberbank-housing/baseline_plr-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/baseline_fidelity/sberbank-housing/baseline_plr-evaluation/0.toml new file mode 100644 index 000000000..a68e82651 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/baseline_fidelity/sberbank-housing/baseline_plr-evaluation/0.toml @@ -0,0 +1,34 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/baseline_fidelity/sberbank-housing/baseline_plr-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/baseline_fidelity/sberbank-housing/baseline_plr-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/baseline_fidelity/sberbank-housing/baseline_plr-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/baseline_fidelity/sberbank-housing/baseline_plr-evaluation/0/report.json new file mode 100644 index 000000000..cfb76d49d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/baseline_fidelity/sberbank-housing/baseline_plr-evaluation/0/report.json @@ -0,0 +1,186 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22224988999004666, + "mae": 0.11853151023387909, + "r2": 0.6844290494918823, + "score": -0.22224988999004666 + }, + "val": { + "rmse": 0.23452539387100468, + "mae": 0.12745676934719086, + "r2": 0.6326068639755249, + "score": -0.23452539387100468 + }, + "test": { + "rmse": 0.23846659721529556, + "mae": 0.15488465130329132, + "r2": 0.6114072799682617, + "score": -0.23846659721529556 + } + }, + "time": "0:00:48.779291", + "diversity": { + "train": { + "mean_centered_corr": -0.018623020499944687, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05047355964779854 + }, + "val": { + "mean_centered_corr": -0.01870153285562992, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06864207983016968 + }, + "test": { + "mean_centered_corr": -0.019026711583137512, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09529124945402145 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/baseline_fidelity/sberbank-housing/baseline_plr-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/baseline_fidelity/sberbank-housing/baseline_plr-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/baseline_fidelity/sberbank-housing/baseline_plr-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/baseline_fidelity/sberbank-housing/baseline_plr-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..e6d5a6174 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/baseline_fidelity/sberbank-housing/baseline_plr-greedy-heads-evaluation/0/report.json @@ -0,0 +1,199 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22302489965783162, + "mae": 0.11715596914291382, + "r2": 0.6822243928909302, + "score": -0.22302489965783162 + }, + "val": { + "rmse": 0.23257941028161666, + "mae": 0.12573255598545074, + "r2": 0.6386785507202148, + "score": -0.23257941028161666 + }, + "test": { + "rmse": 0.23549883338311248, + "mae": 0.14942243695259094, + "r2": 0.6210193037986755, + "score": -0.23549883338311248 + } + }, + "time": "0:00:48.779291", + "diversity": { + "train": { + "mean_centered_corr": -0.018623020499944687, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05047355964779854 + }, + "val": { + "mean_centered_corr": -0.01870153285562992, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06864207983016968 + }, + "test": { + "mean_centered_corr": -0.019026711583137512, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09529124945402145 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 19, + 21, + 12, + 16, + 18, + 8, + 22, + 19, + 29 + ], + "head_selection_time": "0:00:00.160128" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-best-head-evaluation/0/report.json new file mode 100644 index 000000000..dca12e41a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-best-head-evaluation/0/report.json @@ -0,0 +1,147 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44650602179756516, + "mae": 0.34757497906684875, + "r2": 0.5420529246330261, + "score": -0.44650602179756516 + }, + "val": { + "rmse": 0.4652274815891859, + "mae": 0.3617282807826996, + "r2": 0.503640353679657, + "score": -0.4652274815891859 + }, + "test": { + "rmse": 0.4820660657381116, + "mae": 0.3761262893676758, + "r2": 0.48187607526779175, + "score": -0.4820660657381116 + } + }, + "time": "0:09:50.236903", + "diversity": { + "train": { + "mean_centered_corr": -0.020426616072654724, + "mean_pairwise_disagreement": NaN, + "member_std": 0.051189813762903214 + }, + "val": { + "mean_centered_corr": -0.02039431221783161, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05273108929395676 + }, + "test": { + "mean_centered_corr": -0.020482471212744713, + "mean_pairwise_disagreement": NaN, + "member_std": 0.053407639265060425 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 27 + ], + "head_selection_time": "0:00:00.028691" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-best-head-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-best-head-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-best-head-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-best-head-evaluation/1/report.json new file mode 100644 index 000000000..f3a0fb9e6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-best-head-evaluation/1/report.json @@ -0,0 +1,147 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 19092, + "metrics": { + "train": { + "rmse": 0.4486631357443507, + "mae": 0.3495190143585205, + "r2": 0.5376174449920654, + "score": -0.4486631357443507 + }, + "val": { + "rmse": 0.46461960653787665, + "mae": 0.3615076541900635, + "r2": 0.5049366354942322, + "score": -0.46461960653787665 + }, + "test": { + "rmse": 0.4822480664686614, + "mae": 0.37663528323173523, + "r2": 0.4814847707748413, + "score": -0.4822480664686614 + } + }, + "time": "0:08:23.758542", + "diversity": { + "train": { + "mean_centered_corr": -0.020497672259807587, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04839831218123436 + }, + "val": { + "mean_centered_corr": -0.020458761602640152, + "mean_pairwise_disagreement": NaN, + "member_std": 0.049754440784454346 + }, + "test": { + "mean_centered_corr": -0.02041211724281311, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05034574493765831 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 5 + ], + "head_selection_time": "0:00:00.027928" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-best-head-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-best-head-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-best-head-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-best-head-evaluation/2/report.json new file mode 100644 index 000000000..0523e5aa6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-best-head-evaluation/2/report.json @@ -0,0 +1,147 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 28638, + "metrics": { + "train": { + "rmse": 0.4464703615247226, + "mae": 0.34800708293914795, + "r2": 0.5421260595321655, + "score": -0.4464703615247226 + }, + "val": { + "rmse": 0.46567157154030486, + "mae": 0.3625640869140625, + "r2": 0.5026922821998596, + "score": -0.46567157154030486 + }, + "test": { + "rmse": 0.48281151175783615, + "mae": 0.3768390119075775, + "r2": 0.48027241230010986, + "score": -0.48281151175783615 + } + }, + "time": "0:11:51.682809", + "diversity": { + "train": { + "mean_centered_corr": -0.020397048443555832, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05753684788942337 + }, + "val": { + "mean_centered_corr": -0.02034812979400158, + "mean_pairwise_disagreement": NaN, + "member_std": 0.059377871453762054 + }, + "test": { + "mean_centered_corr": -0.02046407200396061, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05992777645587921 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.029046" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-evaluation/0.toml new file mode 100644 index 000000000..78c613dfb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-evaluation/0.toml @@ -0,0 +1,33 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-evaluation/0/report.json new file mode 100644 index 000000000..2c476e8cd --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-evaluation/0/report.json @@ -0,0 +1,143 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44389690465868414, + "mae": 0.34541288018226624, + "r2": 0.5473892688751221, + "score": -0.44389690465868414 + }, + "val": { + "rmse": 0.4627596029926704, + "mae": 0.35959550738334656, + "r2": 0.5088924169540405, + "score": -0.4627596029926704 + }, + "test": { + "rmse": 0.4802386655626689, + "mae": 0.37469902634620667, + "r2": 0.4857967495918274, + "score": -0.4802386655626689 + } + }, + "time": "0:09:50.236903", + "diversity": { + "train": { + "mean_centered_corr": -0.020426616072654724, + "mean_pairwise_disagreement": NaN, + "member_std": 0.051189813762903214 + }, + "val": { + "mean_centered_corr": -0.02039431221783161, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05273108929395676 + }, + "test": { + "mean_centered_corr": -0.020482471212744713, + "mean_pairwise_disagreement": NaN, + "member_std": 0.053407639265060425 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-evaluation/1.toml new file mode 100644 index 000000000..bc39632dd --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-evaluation/1.toml @@ -0,0 +1,33 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 1 +head_selection = true + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-evaluation/1/report.json new file mode 100644 index 000000000..97b04029e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-evaluation/1/report.json @@ -0,0 +1,143 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 19092, + "metrics": { + "train": { + "rmse": 0.4455413230457775, + "mae": 0.34672579169273376, + "r2": 0.5440295934677124, + "score": -0.4455413230457775 + }, + "val": { + "rmse": 0.46274572430329897, + "mae": 0.35959213972091675, + "r2": 0.5089218616485596, + "score": -0.46274572430329897 + }, + "test": { + "rmse": 0.4804413950506519, + "mae": 0.3748830258846283, + "r2": 0.48536252975463867, + "score": -0.4804413950506519 + } + }, + "time": "0:08:23.758542", + "diversity": { + "train": { + "mean_centered_corr": -0.020497672259807587, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04839831218123436 + }, + "val": { + "mean_centered_corr": -0.020458761602640152, + "mean_pairwise_disagreement": NaN, + "member_std": 0.049754440784454346 + }, + "test": { + "mean_centered_corr": -0.02041211724281311, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05034574493765831 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-evaluation/2.toml new file mode 100644 index 000000000..fc2b3587f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-evaluation/2.toml @@ -0,0 +1,33 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 2 +head_selection = true + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-evaluation/2/report.json new file mode 100644 index 000000000..ef23f4c76 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-evaluation/2/report.json @@ -0,0 +1,143 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 28638, + "metrics": { + "train": { + "rmse": 0.4415416847338914, + "mae": 0.3435233533382416, + "r2": 0.5521793961524963, + "score": -0.4415416847338914 + }, + "val": { + "rmse": 0.4626209262038764, + "mae": 0.3592834174633026, + "r2": 0.5091867446899414, + "score": -0.4626209262038764 + }, + "test": { + "rmse": 0.48046188039443216, + "mae": 0.37472644448280334, + "r2": 0.485318660736084, + "score": -0.48046188039443216 + } + }, + "time": "0:11:51.682809", + "diversity": { + "train": { + "mean_centered_corr": -0.020397048443555832, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05753684788942337 + }, + "val": { + "mean_centered_corr": -0.02034812979400158, + "mean_pairwise_disagreement": NaN, + "member_std": 0.059377871453762054 + }, + "test": { + "mean_centered_corr": -0.02046407200396061, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05992777645587921 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..81f6192e3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/0/report.json @@ -0,0 +1,157 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44394242184625543, + "mae": 0.3455282151699066, + "r2": 0.547296404838562, + "score": -0.44394242184625543 + }, + "val": { + "rmse": 0.4625169879783321, + "mae": 0.3595449924468994, + "r2": 0.5094072818756104, + "score": -0.4625169879783321 + }, + "test": { + "rmse": 0.47990786370986477, + "mae": 0.3745970129966736, + "r2": 0.4865049123764038, + "score": -0.47990786370986477 + } + }, + "time": "0:09:50.236903", + "diversity": { + "train": { + "mean_centered_corr": -0.020426616072654724, + "mean_pairwise_disagreement": NaN, + "member_std": 0.051189813762903214 + }, + "val": { + "mean_centered_corr": -0.02039431221783161, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05273108929395676 + }, + "test": { + "mean_centered_corr": -0.020482471212744713, + "mean_pairwise_disagreement": NaN, + "member_std": 0.053407639265060425 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 27, + 17, + 5, + 3, + 14, + 4, + 22, + 6, + 24, + 10, + 16 + ], + "head_selection_time": "0:00:00.401983" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/1/report.json new file mode 100644 index 000000000..cddf7150d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/1/report.json @@ -0,0 +1,162 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 19092, + "metrics": { + "train": { + "rmse": 0.44558913020992225, + "mae": 0.34685587882995605, + "r2": 0.5439317226409912, + "score": -0.44558913020992225 + }, + "val": { + "rmse": 0.4624852364993238, + "mae": 0.3594231605529785, + "r2": 0.5094746351242065, + "score": -0.4624852364993238 + }, + "test": { + "rmse": 0.4800568652741734, + "mae": 0.3746061325073242, + "r2": 0.48618602752685547, + "score": -0.4800568652741734 + } + }, + "time": "0:08:23.758542", + "diversity": { + "train": { + "mean_centered_corr": -0.020497672259807587, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04839831218123436 + }, + "val": { + "mean_centered_corr": -0.020458761602640152, + "mean_pairwise_disagreement": NaN, + "member_std": 0.049754440784454346 + }, + "test": { + "mean_centered_corr": -0.02041211724281311, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05034574493765831 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 5, + 15, + 11, + 9, + 30, + 25, + 6, + 24, + 4, + 13, + 27, + 10, + 9, + 16, + 0, + 28 + ], + "head_selection_time": "0:00:00.658431" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/2/report.json new file mode 100644 index 000000000..a9fa005f6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/2/report.json @@ -0,0 +1,157 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 28638, + "metrics": { + "train": { + "rmse": 0.4416625198883345, + "mae": 0.34376394748687744, + "r2": 0.5519342422485352, + "score": -0.4416625198883345 + }, + "val": { + "rmse": 0.46240787039558773, + "mae": 0.35923537611961365, + "r2": 0.5096387267112732, + "score": -0.46240787039558773 + }, + "test": { + "rmse": 0.480411417572094, + "mae": 0.3748789131641388, + "r2": 0.48542678356170654, + "score": -0.480411417572094 + } + }, + "time": "0:11:51.682809", + "diversity": { + "train": { + "mean_centered_corr": -0.020397048443555832, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05753684788942337 + }, + "val": { + "mean_centered_corr": -0.02034812979400158, + "mean_pairwise_disagreement": NaN, + "member_std": 0.059377871453762054 + }, + "test": { + "mean_centered_corr": -0.02046407200396061, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05992777645587921 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 13, + 25, + 19, + 23, + 6, + 29, + 7, + 31, + 21, + 3, + 2 + ], + "head_selection_time": "0:00:00.397654" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..222652a2b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-best-head-evaluation/0/report.json @@ -0,0 +1,400 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.45199979641686133, + "mae": 0.3517935872077942, + "r2": 0.5307145118713379, + "score": -0.45199979641686133 + }, + "val": { + "rmse": 0.4638214410008231, + "mae": 0.360493928194046, + "r2": 0.5066360831260681, + "score": -0.4638214410008231 + }, + "test": { + "rmse": 0.4819659035779245, + "mae": 0.37611493468284607, + "r2": 0.48209136724472046, + "score": -0.4819659035779245 + } + }, + "time": "0:10:10.466567", + "diversity": { + "train": { + "mean_centered_corr": -0.005496397614479065, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04279099032282829 + }, + "val": { + "mean_centered_corr": -0.005842795129865408, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04372712969779968 + }, + "test": { + "mean_centered_corr": -0.005908838007599115, + "mean_pairwise_disagreement": NaN, + "member_std": 0.044375959783792496 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31 + ], + "head_selection_time": "0:00:00.027383" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-best-head-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-best-head-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-best-head-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-best-head-evaluation/1/report.json new file mode 100644 index 000000000..62ae87d2c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-best-head-evaluation/1/report.json @@ -0,0 +1,400 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 14652, + "metrics": { + "train": { + "rmse": 0.45153318527792236, + "mae": 0.351412832736969, + "r2": 0.5316829681396484, + "score": -0.45153318527792236 + }, + "val": { + "rmse": 0.4640342331314561, + "mae": 0.3607448935508728, + "r2": 0.5061832666397095, + "score": -0.4640342331314561 + }, + "test": { + "rmse": 0.4812057394258983, + "mae": 0.3755890727043152, + "r2": 0.4837237596511841, + "score": -0.4812057394258983 + } + }, + "time": "0:08:47.757971", + "diversity": { + "train": { + "mean_centered_corr": -0.011369073763489723, + "mean_pairwise_disagreement": NaN, + "member_std": 0.042549557983875275 + }, + "val": { + "mean_centered_corr": -0.011497393250465393, + "mean_pairwise_disagreement": NaN, + "member_std": 0.043487798422575 + }, + "test": { + "mean_centered_corr": -0.011657103896141052, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0439562126994133 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25 + ], + "head_selection_time": "0:00:00.027507" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-best-head-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-best-head-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-best-head-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-best-head-evaluation/2/report.json new file mode 100644 index 000000000..78d8c3953 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-best-head-evaluation/2/report.json @@ -0,0 +1,400 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 12876, + "metrics": { + "train": { + "rmse": 0.452644751353011, + "mae": 0.35248246788978577, + "r2": 0.529374361038208, + "score": -0.452644751353011 + }, + "val": { + "rmse": 0.46399304740420017, + "mae": 0.3608178496360779, + "r2": 0.5062709450721741, + "score": -0.46399304740420017 + }, + "test": { + "rmse": 0.4807545037031361, + "mae": 0.3752909302711487, + "r2": 0.4846915602684021, + "score": -0.4807545037031361 + } + }, + "time": "0:07:48.322244", + "diversity": { + "train": { + "mean_centered_corr": -0.014557692222297192, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04135805740952492 + }, + "val": { + "mean_centered_corr": -0.014966246671974659, + "mean_pairwise_disagreement": NaN, + "member_std": 0.042381998151540756 + }, + "test": { + "mean_centered_corr": -0.014708246104419231, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04306395351886749 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 23 + ], + "head_selection_time": "0:00:00.028302" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/0.toml new file mode 100644 index 000000000..784c17199 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/0.toml @@ -0,0 +1,44 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 + +[cf_fisd] +lambda = 0.05 +variant = "raw" +dataset_name = "cooking-time" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/0/report.json new file mode 100644 index 000000000..f58c2dd92 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/0/report.json @@ -0,0 +1,396 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.4482607702587857, + "mae": 0.34888917207717896, + "r2": 0.5384464263916016, + "score": -0.4482607702587857 + }, + "val": { + "rmse": 0.4632019090821483, + "mae": 0.3600703775882721, + "r2": 0.5079531669616699, + "score": -0.4632019090821483 + }, + "test": { + "rmse": 0.4808496810468957, + "mae": 0.3752937614917755, + "r2": 0.48448747396469116, + "score": -0.4808496810468957 + } + }, + "time": "0:10:10.466567", + "diversity": { + "train": { + "mean_centered_corr": -0.005496397614479065, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04279099032282829 + }, + "val": { + "mean_centered_corr": -0.005842795129865408, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04372712969779968 + }, + "test": { + "mean_centered_corr": -0.005908838007599115, + "mean_pairwise_disagreement": NaN, + "member_std": 0.044375959783792496 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/1.toml new file mode 100644 index 000000000..7ab1c4267 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/1.toml @@ -0,0 +1,44 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 1 +head_selection = true + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 + +[cf_fisd] +lambda = 0.05 +variant = "raw" +dataset_name = "cooking-time" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/1/report.json new file mode 100644 index 000000000..da30cf68e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/1/report.json @@ -0,0 +1,396 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 14652, + "metrics": { + "train": { + "rmse": 0.449091304447225, + "mae": 0.3495481312274933, + "r2": 0.5367345213890076, + "score": -0.449091304447225 + }, + "val": { + "rmse": 0.4632797054631699, + "mae": 0.36018165946006775, + "r2": 0.5077878832817078, + "score": -0.4632797054631699 + }, + "test": { + "rmse": 0.48088074674610853, + "mae": 0.37531033158302307, + "r2": 0.48442089557647705, + "score": -0.48088074674610853 + } + }, + "time": "0:08:47.757971", + "diversity": { + "train": { + "mean_centered_corr": -0.011369073763489723, + "mean_pairwise_disagreement": NaN, + "member_std": 0.042549557983875275 + }, + "val": { + "mean_centered_corr": -0.011497393250465393, + "mean_pairwise_disagreement": NaN, + "member_std": 0.043487798422575 + }, + "test": { + "mean_centered_corr": -0.011657103896141052, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0439562126994133 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/2.toml new file mode 100644 index 000000000..516923524 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/2.toml @@ -0,0 +1,44 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 2 +head_selection = true + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 + +[cf_fisd] +lambda = 0.05 +variant = "raw" +dataset_name = "cooking-time" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/2/report.json new file mode 100644 index 000000000..f9241e83b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/2/report.json @@ -0,0 +1,396 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 12876, + "metrics": { + "train": { + "rmse": 0.4496297034631907, + "mae": 0.3498455882072449, + "r2": 0.5356230735778809, + "score": -0.4496297034631907 + }, + "val": { + "rmse": 0.46327719662519024, + "mae": 0.359968900680542, + "r2": 0.5077931880950928, + "score": -0.46327719662519024 + }, + "test": { + "rmse": 0.48070024325509664, + "mae": 0.3750893771648407, + "r2": 0.4848078489303589, + "score": -0.48070024325509664 + } + }, + "time": "0:07:48.322244", + "diversity": { + "train": { + "mean_centered_corr": -0.014557692222297192, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04135805740952492 + }, + "val": { + "mean_centered_corr": -0.014966246671974659, + "mean_pairwise_disagreement": NaN, + "member_std": 0.042381998151540756 + }, + "test": { + "mean_centered_corr": -0.014708246104419231, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04306395351886749 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..0f7f787a9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,413 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.44782199154768965, + "mae": 0.34862202405929565, + "r2": 0.5393495559692383, + "score": -0.44782199154768965 + }, + "val": { + "rmse": 0.4628482268025946, + "mae": 0.35992974042892456, + "r2": 0.5087043046951294, + "score": -0.4628482268025946 + }, + "test": { + "rmse": 0.4803119031248436, + "mae": 0.3749502897262573, + "r2": 0.48563992977142334, + "score": -0.4803119031248436 + } + }, + "time": "0:10:10.466567", + "diversity": { + "train": { + "mean_centered_corr": -0.005496397614479065, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04279099032282829 + }, + "val": { + "mean_centered_corr": -0.005842795129865408, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04372712969779968 + }, + "test": { + "mean_centered_corr": -0.005908838007599115, + "mean_pairwise_disagreement": NaN, + "member_std": 0.044375959783792496 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31, + 27, + 14, + 2, + 21, + 25, + 2, + 12, + 22, + 14, + 27, + 2, + 19, + 24 + ], + "head_selection_time": "0:00:00.542336" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/1/report.json new file mode 100644 index 000000000..5784ed8d0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/1/report.json @@ -0,0 +1,412 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 14652, + "metrics": { + "train": { + "rmse": 0.448658967573918, + "mae": 0.34933236241340637, + "r2": 0.5376260280609131, + "score": -0.448658967573918 + }, + "val": { + "rmse": 0.4629984701854322, + "mae": 0.36006447672843933, + "r2": 0.5083853006362915, + "score": -0.4629984701854322 + }, + "test": { + "rmse": 0.4803911006799199, + "mae": 0.375040739774704, + "r2": 0.4854702949523926, + "score": -0.4803911006799199 + } + }, + "time": "0:08:47.757971", + "diversity": { + "train": { + "mean_centered_corr": -0.011369073763489723, + "mean_pairwise_disagreement": NaN, + "member_std": 0.042549557983875275 + }, + "val": { + "mean_centered_corr": -0.011497393250465393, + "mean_pairwise_disagreement": NaN, + "member_std": 0.043487798422575 + }, + "test": { + "mean_centered_corr": -0.011657103896141052, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0439562126994133 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25, + 0, + 18, + 26, + 14, + 10, + 19, + 27, + 16, + 15, + 1, + 24, + 0 + ], + "head_selection_time": "0:00:00.482246" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/2/report.json new file mode 100644 index 000000000..88ca26cfd --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/2/report.json @@ -0,0 +1,410 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 12876, + "metrics": { + "train": { + "rmse": 0.4491811321927657, + "mae": 0.34958937764167786, + "r2": 0.5365492105484009, + "score": -0.4491811321927657 + }, + "val": { + "rmse": 0.46301623543457804, + "mae": 0.359862744808197, + "r2": 0.5083475708961487, + "score": -0.46301623543457804 + }, + "test": { + "rmse": 0.4803092971079454, + "mae": 0.3748699426651001, + "r2": 0.4856455326080322, + "score": -0.4803092971079454 + } + }, + "time": "0:07:48.322244", + "diversity": { + "train": { + "mean_centered_corr": -0.014557692222297192, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04135805740952492 + }, + "val": { + "mean_centered_corr": -0.014966246671974659, + "mean_pairwise_disagreement": NaN, + "member_std": 0.042381998151540756 + }, + "test": { + "mean_centered_corr": -0.014708246104419231, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04306395351886749 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 23, + 2, + 11, + 22, + 15, + 0, + 12, + 25, + 18, + 8, + 21 + ], + "head_selection_time": "0:00:00.407629" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-best-head-evaluation/0/report.json new file mode 100644 index 000000000..f1c458896 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-best-head-evaluation/0/report.json @@ -0,0 +1,166 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_esam-evaluation/_esam_diagnostics/rla_esam-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44621279542701975, + "mae": 0.3473000228404999, + "r2": 0.5426542162895203, + "score": -0.44621279542701975 + }, + "val": { + "rmse": 0.46492255368357066, + "mae": 0.36141437292099, + "r2": 0.5042908191680908, + "score": -0.46492255368357066 + }, + "test": { + "rmse": 0.4825495500767423, + "mae": 0.37668848037719727, + "r2": 0.4808362126350403, + "score": -0.4825495500767423 + } + }, + "time": "0:37:31.095542", + "diversity": { + "train": { + "mean_centered_corr": -0.02036323957145214, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05055415257811546 + }, + "val": { + "mean_centered_corr": -0.02048061601817608, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05214421823620796 + }, + "test": { + "mean_centered_corr": -0.020323824137449265, + "mean_pairwise_disagreement": NaN, + "member_std": 0.052737440913915634 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 14 + ], + "head_selection_time": "0:00:00.027759" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-best-head-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-best-head-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-best-head-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-best-head-evaluation/1/report.json new file mode 100644 index 000000000..07656ccee --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-best-head-evaluation/1/report.json @@ -0,0 +1,166 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-evaluation/_esam_diagnostics/best_combined-evaluation__1.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 19092, + "metrics": { + "train": { + "rmse": 0.44843356300247367, + "mae": 0.34928280115127563, + "r2": 0.5380905866622925, + "score": -0.44843356300247367 + }, + "val": { + "rmse": 0.46452431161307095, + "mae": 0.361534982919693, + "r2": 0.5051397085189819, + "score": -0.46452431161307095 + }, + "test": { + "rmse": 0.48248902135425475, + "mae": 0.37688297033309937, + "r2": 0.4809664487838745, + "score": -0.48248902135425475 + } + }, + "time": "0:32:03.279613", + "diversity": { + "train": { + "mean_centered_corr": -0.020463576540350914, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04766561836004257 + }, + "val": { + "mean_centered_corr": -0.020318932831287384, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048978473991155624 + }, + "test": { + "mean_centered_corr": -0.020468220114707947, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04966771602630615 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 5 + ], + "head_selection_time": "0:00:00.027757" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-best-head-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-best-head-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-best-head-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-best-head-evaluation/2/report.json new file mode 100644 index 000000000..fa7f560b8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-best-head-evaluation/2/report.json @@ -0,0 +1,166 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-evaluation/_esam_diagnostics/best_combined-evaluation__2.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 29526, + "metrics": { + "train": { + "rmse": 0.44484696398137763, + "mae": 0.3464239537715912, + "r2": 0.5454497337341309, + "score": -0.44484696398137763 + }, + "val": { + "rmse": 0.46544124766603506, + "mae": 0.3615325093269348, + "r2": 0.5031840801239014, + "score": -0.46544124766603506 + }, + "test": { + "rmse": 0.4832114117959526, + "mae": 0.37725383043289185, + "r2": 0.47941112518310547, + "score": -0.4832114117959526 + } + }, + "time": "0:46:36.662355", + "diversity": { + "train": { + "mean_centered_corr": -0.0203552283346653, + "mean_pairwise_disagreement": NaN, + "member_std": 0.057090889662504196 + }, + "val": { + "mean_centered_corr": -0.020306019112467766, + "mean_pairwise_disagreement": NaN, + "member_std": 0.058967314660549164 + }, + "test": { + "mean_centered_corr": -0.020541546866297722, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0595683790743351 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 21 + ], + "head_selection_time": "0:00:00.027626" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-evaluation/0.toml new file mode 100644 index 000000000..0b28045f1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-evaluation/0.toml @@ -0,0 +1,50 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-evaluation/0/report.json new file mode 100644 index 000000000..9520ae39a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-evaluation/0/report.json @@ -0,0 +1,162 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_esam-evaluation/_esam_diagnostics/rla_esam-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4437324533888703, + "mae": 0.3453030586242676, + "r2": 0.5477245450019836, + "score": -0.4437324533888703 + }, + "val": { + "rmse": 0.4627353230636857, + "mae": 0.35955438017845154, + "r2": 0.5089439749717712, + "score": -0.4627353230636857 + }, + "test": { + "rmse": 0.4803077303885999, + "mae": 0.3747372627258301, + "r2": 0.48564887046813965, + "score": -0.4803077303885999 + } + }, + "time": "0:37:31.095542", + "diversity": { + "train": { + "mean_centered_corr": -0.02036323957145214, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05055415257811546 + }, + "val": { + "mean_centered_corr": -0.02048061601817608, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05214421823620796 + }, + "test": { + "mean_centered_corr": -0.020323824137449265, + "mean_pairwise_disagreement": NaN, + "member_std": 0.052737440913915634 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-evaluation/1.toml new file mode 100644 index 000000000..a6c9a5d12 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-evaluation/1.toml @@ -0,0 +1,50 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 1 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-evaluation/1/report.json new file mode 100644 index 000000000..374bdcb1a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-evaluation/1/report.json @@ -0,0 +1,162 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-evaluation/_esam_diagnostics/best_combined-evaluation__1.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 19092, + "metrics": { + "train": { + "rmse": 0.44556993438537174, + "mae": 0.34677401185035706, + "r2": 0.543971061706543, + "score": -0.44556993438537174 + }, + "val": { + "rmse": 0.4627920762114039, + "mae": 0.35961198806762695, + "r2": 0.5088235139846802, + "score": -0.4627920762114039 + }, + "test": { + "rmse": 0.4804951265119692, + "mae": 0.37489259243011475, + "r2": 0.4852474331855774, + "score": -0.4804951265119692 + } + }, + "time": "0:32:03.279613", + "diversity": { + "train": { + "mean_centered_corr": -0.020463576540350914, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04766561836004257 + }, + "val": { + "mean_centered_corr": -0.020318932831287384, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048978473991155624 + }, + "test": { + "mean_centered_corr": -0.020468220114707947, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04966771602630615 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-evaluation/2.toml new file mode 100644 index 000000000..c040c0c45 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-evaluation/2.toml @@ -0,0 +1,50 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 2 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-evaluation/2/report.json new file mode 100644 index 000000000..80a5dc885 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-evaluation/2/report.json @@ -0,0 +1,162 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-evaluation/_esam_diagnostics/best_combined-evaluation__2.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 29526, + "metrics": { + "train": { + "rmse": 0.44099685791198107, + "mae": 0.3432556390762329, + "r2": 0.5532838702201843, + "score": -0.44099685791198107 + }, + "val": { + "rmse": 0.4627156309124574, + "mae": 0.35954052209854126, + "r2": 0.5089857578277588, + "score": -0.4627156309124574 + }, + "test": { + "rmse": 0.4803808798726675, + "mae": 0.3748488426208496, + "r2": 0.48549216985702515, + "score": -0.4803808798726675 + } + }, + "time": "0:46:36.662355", + "diversity": { + "train": { + "mean_centered_corr": -0.0203552283346653, + "mean_pairwise_disagreement": NaN, + "member_std": 0.057090889662504196 + }, + "val": { + "mean_centered_corr": -0.020306019112467766, + "mean_pairwise_disagreement": NaN, + "member_std": 0.058967314660549164 + }, + "test": { + "mean_centered_corr": -0.020541546866297722, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0595683790743351 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..03c3c5cda --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/0/report.json @@ -0,0 +1,178 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_esam-evaluation/_esam_diagnostics/rla_esam-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4439349534537547, + "mae": 0.3455897867679596, + "r2": 0.5473116636276245, + "score": -0.4439349534537547 + }, + "val": { + "rmse": 0.4624234187671886, + "mae": 0.3594515919685364, + "r2": 0.5096057653427124, + "score": -0.4624234187671886 + }, + "test": { + "rmse": 0.4800586966547807, + "mae": 0.37484052777290344, + "r2": 0.4861820936203003, + "score": -0.4800586966547807 + } + }, + "time": "0:37:31.095542", + "diversity": { + "train": { + "mean_centered_corr": -0.02036323957145214, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05055415257811546 + }, + "val": { + "mean_centered_corr": -0.02048061601817608, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05214421823620796 + }, + "test": { + "mean_centered_corr": -0.020323824137449265, + "mean_pairwise_disagreement": NaN, + "member_std": 0.052737440913915634 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 14, + 3, + 17, + 24, + 5, + 10, + 4, + 27, + 31, + 7, + 17, + 3, + 24 + ], + "head_selection_time": "0:00:00.494060" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/1/report.json new file mode 100644 index 000000000..bf8446d82 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/1/report.json @@ -0,0 +1,180 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-evaluation/_esam_diagnostics/best_combined-evaluation__1.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 19092, + "metrics": { + "train": { + "rmse": 0.44579561696398656, + "mae": 0.34707847237586975, + "r2": 0.5435090065002441, + "score": -0.44579561696398656 + }, + "val": { + "rmse": 0.46250972286520653, + "mae": 0.35948577523231506, + "r2": 0.5094226598739624, + "score": -0.46250972286520653 + }, + "test": { + "rmse": 0.4801020579816987, + "mae": 0.37471047043800354, + "r2": 0.486089289188385, + "score": -0.4801020579816987 + } + }, + "time": "0:32:03.279613", + "diversity": { + "train": { + "mean_centered_corr": -0.020463576540350914, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04766561836004257 + }, + "val": { + "mean_centered_corr": -0.020318932831287384, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048978473991155624 + }, + "test": { + "mean_centered_corr": -0.020468220114707947, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04966771602630615 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 5, + 13, + 9, + 6, + 25, + 30, + 15, + 24, + 14, + 8, + 13, + 3, + 4, + 9, + 27 + ], + "head_selection_time": "0:00:00.593938" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/2/report.json new file mode 100644 index 000000000..e9da28fe3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/2/report.json @@ -0,0 +1,178 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-evaluation/_esam_diagnostics/best_combined-evaluation__2.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 29526, + "metrics": { + "train": { + "rmse": 0.44166110285667354, + "mae": 0.3438146710395813, + "r2": 0.5519371628761292, + "score": -0.44166110285667354 + }, + "val": { + "rmse": 0.4623945128795896, + "mae": 0.3593231737613678, + "r2": 0.5096670389175415, + "score": -0.4623945128795896 + }, + "test": { + "rmse": 0.4799174115040007, + "mae": 0.37446993589401245, + "r2": 0.48648446798324585, + "score": -0.4799174115040007 + } + }, + "time": "0:46:36.662355", + "diversity": { + "train": { + "mean_centered_corr": -0.0203552283346653, + "mean_pairwise_disagreement": NaN, + "member_std": 0.057090889662504196 + }, + "val": { + "mean_centered_corr": -0.020306019112467766, + "mean_pairwise_disagreement": NaN, + "member_std": 0.058967314660549164 + }, + "test": { + "mean_centered_corr": -0.020541546866297722, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0595683790743351 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 21, + 12, + 5, + 25, + 16, + 7, + 29, + 0, + 8, + 22, + 26, + 3, + 2 + ], + "head_selection_time": "0:00:00.485946" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..ef69ab721 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-best-head-evaluation/0/report.json @@ -0,0 +1,157 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.0025-evaluation/_esam_diagnostics/esam_rho0.0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44801757104722606, + "mae": 0.348697692155838, + "r2": 0.5389471054077148, + "score": -0.44801757104722606 + }, + "val": { + "rmse": 0.46513258351328474, + "mae": 0.36148601770401, + "r2": 0.503842830657959, + "score": -0.46513258351328474 + }, + "test": { + "rmse": 0.48253251942068065, + "mae": 0.37657663226127625, + "r2": 0.48087286949157715, + "score": -0.48253251942068065 + } + }, + "time": "0:15:54.055312", + "diversity": { + "train": { + "mean_centered_corr": -0.020338764414191246, + "mean_pairwise_disagreement": NaN, + "member_std": 0.050871677696704865 + }, + "val": { + "mean_centered_corr": -0.02030899003148079, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05245138704776764 + }, + "test": { + "mean_centered_corr": -0.020395034924149513, + "mean_pairwise_disagreement": NaN, + "member_std": 0.053137533366680145 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 7 + ], + "head_selection_time": "0:00:00.028441" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-best-head-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-best-head-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-best-head-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-best-head-evaluation/1/report.json new file mode 100644 index 000000000..635004bc0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-best-head-evaluation/1/report.json @@ -0,0 +1,157 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__1.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 19092, + "metrics": { + "train": { + "rmse": 0.44870194270769953, + "mae": 0.349582701921463, + "r2": 0.5375374555587769, + "score": -0.44870194270769953 + }, + "val": { + "rmse": 0.46441974054703444, + "mae": 0.3615182936191559, + "r2": 0.5053625106811523, + "score": -0.46441974054703444 + }, + "test": { + "rmse": 0.48248829558115747, + "mae": 0.3769335150718689, + "r2": 0.48096799850463867, + "score": -0.48248829558115747 + } + }, + "time": "0:13:31.147489", + "diversity": { + "train": { + "mean_centered_corr": -0.02047574520111084, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0481378473341465 + }, + "val": { + "mean_centered_corr": -0.020433666184544563, + "mean_pairwise_disagreement": NaN, + "member_std": 0.049497347325086594 + }, + "test": { + "mean_centered_corr": -0.02056153491139412, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05010291561484337 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 5 + ], + "head_selection_time": "0:00:00.028702" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-best-head-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-best-head-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-best-head-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-best-head-evaluation/2/report.json new file mode 100644 index 000000000..2f3742352 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-best-head-evaluation/2/report.json @@ -0,0 +1,157 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__2.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 29526, + "metrics": { + "train": { + "rmse": 0.44640390600457625, + "mae": 0.34810319542884827, + "r2": 0.5422623753547668, + "score": -0.44640390600457625 + }, + "val": { + "rmse": 0.4655328979544962, + "mae": 0.36213234066963196, + "r2": 0.5029884576797485, + "score": -0.4655328979544962 + }, + "test": { + "rmse": 0.48239242216697287, + "mae": 0.3769451081752777, + "r2": 0.4811742901802063, + "score": -0.48239242216697287 + } + }, + "time": "0:19:42.465328", + "diversity": { + "train": { + "mean_centered_corr": -0.02041659690439701, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05716099962592125 + }, + "val": { + "mean_centered_corr": -0.020310183987021446, + "mean_pairwise_disagreement": NaN, + "member_std": 0.059111326932907104 + }, + "test": { + "mean_centered_corr": -0.02049497328698635, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05964430794119835 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 12 + ], + "head_selection_time": "0:00:00.027416" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-evaluation/0.toml new file mode 100644 index 000000000..1d55eda06 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-evaluation/0.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.0025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-evaluation/0/report.json new file mode 100644 index 000000000..7b9d04d15 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-evaluation/0/report.json @@ -0,0 +1,153 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.0025-evaluation/_esam_diagnostics/esam_rho0.0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4439953515264817, + "mae": 0.34551990032196045, + "r2": 0.547188401222229, + "score": -0.4439953515264817 + }, + "val": { + "rmse": 0.4627563346148938, + "mae": 0.3595946729183197, + "r2": 0.5088993906974792, + "score": -0.4627563346148938 + }, + "test": { + "rmse": 0.4802735094812688, + "mae": 0.3747326135635376, + "r2": 0.4857221841812134, + "score": -0.4802735094812688 + } + }, + "time": "0:15:54.055312", + "diversity": { + "train": { + "mean_centered_corr": -0.020338764414191246, + "mean_pairwise_disagreement": NaN, + "member_std": 0.050871677696704865 + }, + "val": { + "mean_centered_corr": -0.02030899003148079, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05245138704776764 + }, + "test": { + "mean_centered_corr": -0.020395034924149513, + "mean_pairwise_disagreement": NaN, + "member_std": 0.053137533366680145 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-evaluation/1.toml new file mode 100644 index 000000000..a63507c9a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-evaluation/1.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 1 +head_selection = true +use_esam = true +esam_rho = 0.0025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-evaluation/1/report.json new file mode 100644 index 000000000..6a6a08dbf --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-evaluation/1/report.json @@ -0,0 +1,153 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__1.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 19092, + "metrics": { + "train": { + "rmse": 0.4456571115584819, + "mae": 0.34683623909950256, + "r2": 0.5437926054000854, + "score": -0.4456571115584819 + }, + "val": { + "rmse": 0.4627833664641114, + "mae": 0.3596556782722473, + "r2": 0.5088419914245605, + "score": -0.4627833664641114 + }, + "test": { + "rmse": 0.4804732935000574, + "mae": 0.3749082386493683, + "r2": 0.4852942228317261, + "score": -0.4804732935000574 + } + }, + "time": "0:13:31.147489", + "diversity": { + "train": { + "mean_centered_corr": -0.02047574520111084, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0481378473341465 + }, + "val": { + "mean_centered_corr": -0.020433666184544563, + "mean_pairwise_disagreement": NaN, + "member_std": 0.049497347325086594 + }, + "test": { + "mean_centered_corr": -0.02056153491139412, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05010291561484337 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-evaluation/2.toml new file mode 100644 index 000000000..78470715d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-evaluation/2.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 2 +head_selection = true +use_esam = true +esam_rho = 0.0025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-evaluation/2/report.json new file mode 100644 index 000000000..ac2e0b2b8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-evaluation/2/report.json @@ -0,0 +1,153 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__2.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 29526, + "metrics": { + "train": { + "rmse": 0.4413235510941761, + "mae": 0.3435209393501282, + "r2": 0.5526217818260193, + "score": -0.4413235510941761 + }, + "val": { + "rmse": 0.4626873390860138, + "mae": 0.359529584646225, + "r2": 0.5090458393096924, + "score": -0.4626873390860138 + }, + "test": { + "rmse": 0.4802976784438942, + "mae": 0.3747992217540741, + "r2": 0.48567038774490356, + "score": -0.4802976784438942 + } + }, + "time": "0:19:42.465328", + "diversity": { + "train": { + "mean_centered_corr": -0.02041659690439701, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05716099962592125 + }, + "val": { + "mean_centered_corr": -0.020310183987021446, + "mean_pairwise_disagreement": NaN, + "member_std": 0.059111326932907104 + }, + "test": { + "mean_centered_corr": -0.02049497328698635, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05964430794119835 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..d892e337e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.0025-evaluation/_esam_diagnostics/esam_rho0.0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4440075509733049, + "mae": 0.345533162355423, + "r2": 0.5471635460853577, + "score": -0.4440075509733049 + }, + "val": { + "rmse": 0.46246732196836127, + "mae": 0.35945001244544983, + "r2": 0.5095126032829285, + "score": -0.46246732196836127 + }, + "test": { + "rmse": 0.4800169922135662, + "mae": 0.3746195435523987, + "r2": 0.48627138137817383, + "score": -0.4800169922135662 + } + }, + "time": "0:15:54.055312", + "diversity": { + "train": { + "mean_centered_corr": -0.020338764414191246, + "mean_pairwise_disagreement": NaN, + "member_std": 0.050871677696704865 + }, + "val": { + "mean_centered_corr": -0.02030899003148079, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05245138704776764 + }, + "test": { + "mean_centered_corr": -0.020395034924149513, + "mean_pairwise_disagreement": NaN, + "member_std": 0.053137533366680145 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 7, + 5, + 27, + 17, + 24, + 3, + 14, + 4, + 22, + 5, + 10, + 21, + 17, + 27, + 24, + 16, + 3, + 5, + 12, + 18, + 14, + 27, + 17 + ], + "head_selection_time": "0:00:01.091181" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/1/report.json new file mode 100644 index 000000000..7f7238263 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/1/report.json @@ -0,0 +1,166 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__1.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 19092, + "metrics": { + "train": { + "rmse": 0.4459791714419137, + "mae": 0.3472432792186737, + "r2": 0.543133020401001, + "score": -0.4459791714419137 + }, + "val": { + "rmse": 0.46256428091435436, + "mae": 0.3595920205116272, + "r2": 0.5093069076538086, + "score": -0.46256428091435436 + }, + "test": { + "rmse": 0.4801574722160279, + "mae": 0.3747093081474304, + "r2": 0.4859706163406372, + "score": -0.4801574722160279 + } + }, + "time": "0:13:31.147489", + "diversity": { + "train": { + "mean_centered_corr": -0.02047574520111084, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0481378473341465 + }, + "val": { + "mean_centered_corr": -0.020433666184544563, + "mean_pairwise_disagreement": NaN, + "member_std": 0.049497347325086594 + }, + "test": { + "mean_centered_corr": -0.02056153491139412, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05010291561484337 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 5, + 13, + 9, + 30, + 25, + 6, + 24, + 15, + 18, + 11 + ], + "head_selection_time": "0:00:00.355063" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/2/report.json new file mode 100644 index 000000000..2ca396a30 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/2/report.json @@ -0,0 +1,174 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__2.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 29526, + "metrics": { + "train": { + "rmse": 0.44162036125193554, + "mae": 0.3438238501548767, + "r2": 0.5520198345184326, + "score": -0.44162036125193554 + }, + "val": { + "rmse": 0.462401054726952, + "mae": 0.3593364953994751, + "r2": 0.5096532106399536, + "score": -0.462401054726952 + }, + "test": { + "rmse": 0.48002014306639385, + "mae": 0.3746410012245178, + "r2": 0.48626458644866943, + "score": -0.48002014306639385 + } + }, + "time": "0:19:42.465328", + "diversity": { + "train": { + "mean_centered_corr": -0.02041659690439701, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05716099962592125 + }, + "val": { + "mean_centered_corr": -0.020310183987021446, + "mean_pairwise_disagreement": NaN, + "member_std": 0.059111326932907104 + }, + "test": { + "mean_centered_corr": -0.02049497328698635, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05964430794119835 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 12, + 21, + 29, + 5, + 22, + 16, + 8, + 7, + 25, + 3, + 23, + 26, + 28, + 0, + 31, + 21, + 29, + 18 + ], + "head_selection_time": "0:00:00.787252" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..50ed2f8b2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-best-head-evaluation/0/report.json @@ -0,0 +1,183 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.8019205927848816, + "min_features_per_member": 141.0, + "mean_features_per_member": 153.96875, + "max_features_per_member": 167.0, + "min_members_per_feature": 21.0, + "mean_members_per_feature": 25.66145896911621, + "max_members_per_feature": 31.0, + "mean_pairwise_jaccard": 0.6682771134235742, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44690438220657625, + "mae": 0.3479042053222656, + "r2": 0.5412354469299316, + "score": -0.44690438220657625 + }, + "val": { + "rmse": 0.4652213638497702, + "mae": 0.36172589659690857, + "r2": 0.5036534070968628, + "score": -0.4652213638497702 + }, + "test": { + "rmse": 0.48343941871247514, + "mae": 0.37721583247184753, + "r2": 0.4789196848869324, + "score": -0.48343941871247514 + } + }, + "time": "0:12:07.675037", + "diversity": { + "train": { + "mean_centered_corr": -0.019306395202875137, + "mean_pairwise_disagreement": NaN, + "member_std": 0.057879555970430374 + }, + "val": { + "mean_centered_corr": -0.01832164078950882, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06112360581755638 + }, + "test": { + "mean_centered_corr": -0.01794510707259178, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06266573071479797 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 19 + ], + "head_selection_time": "0:00:00.027886" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-best-head-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-best-head-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-best-head-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-best-head-evaluation/1/report.json new file mode 100644 index 000000000..4197d13eb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-best-head-evaluation/1/report.json @@ -0,0 +1,183 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + }, + "mask_stats": { + "keep_rate_actual": 0.8004557490348816, + "min_features_per_member": 142.0, + "mean_features_per_member": 153.6875, + "max_features_per_member": 165.0, + "min_members_per_feature": 19.0, + "mean_members_per_feature": 25.61458396911621, + "max_members_per_feature": 30.0, + "mean_pairwise_jaccard": 0.6664211145724647, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 18870, + "metrics": { + "train": { + "rmse": 0.4492457506109391, + "mae": 0.3500923812389374, + "r2": 0.5364158153533936, + "score": -0.4492457506109391 + }, + "val": { + "rmse": 0.4651112788369628, + "mae": 0.3617507219314575, + "r2": 0.5038883090019226, + "score": -0.4651112788369628 + }, + "test": { + "rmse": 0.4825797035175412, + "mae": 0.3767949938774109, + "r2": 0.48077136278152466, + "score": -0.4825797035175412 + } + }, + "time": "0:10:14.470810", + "diversity": { + "train": { + "mean_centered_corr": -0.019374029710888863, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05610343813896179 + }, + "val": { + "mean_centered_corr": -0.018558263778686523, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05900688096880913 + }, + "test": { + "mean_centered_corr": -0.018259482458233833, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06036880984902382 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 27 + ], + "head_selection_time": "0:00:00.027978" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-best-head-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-best-head-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-best-head-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-best-head-evaluation/2/report.json new file mode 100644 index 000000000..17110eb64 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-best-head-evaluation/2/report.json @@ -0,0 +1,183 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + }, + "mask_stats": { + "keep_rate_actual": 0.8037109375, + "min_features_per_member": 144.0, + "mean_features_per_member": 154.3125, + "max_features_per_member": 166.0, + "min_members_per_feature": 19.0, + "mean_members_per_feature": 25.71875, + "max_members_per_feature": 31.0, + "mean_pairwise_jaccard": 0.6710767481400324, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 28638, + "metrics": { + "train": { + "rmse": 0.44502781250652484, + "mae": 0.3467985689640045, + "r2": 0.5450800657272339, + "score": -0.44502781250652484 + }, + "val": { + "rmse": 0.4653975929915622, + "mae": 0.3622805178165436, + "r2": 0.5032773017883301, + "score": -0.4653975929915622 + }, + "test": { + "rmse": 0.48330849475451526, + "mae": 0.3774421513080597, + "r2": 0.47920191287994385, + "score": -0.48330849475451526 + } + }, + "time": "0:14:35.891802", + "diversity": { + "train": { + "mean_centered_corr": -0.019855137914419174, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06328506022691727 + }, + "val": { + "mean_centered_corr": -0.01927705854177475, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06649371236562729 + }, + "test": { + "mean_centered_corr": -0.018950071185827255, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06782469898462296 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 19 + ], + "head_selection_time": "0:00:00.028731" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-evaluation/0.toml new file mode 100644 index 000000000..83cf7dea9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-evaluation/0.toml @@ -0,0 +1,46 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.8 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-evaluation/0/report.json new file mode 100644 index 000000000..835fe9a93 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.8019205927848816, + "min_features_per_member": 141.0, + "mean_features_per_member": 153.96875, + "max_features_per_member": 167.0, + "min_members_per_feature": 21.0, + "mean_members_per_feature": 25.66145896911621, + "max_members_per_feature": 31.0, + "mean_pairwise_jaccard": 0.6682771134235742, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4439757847261177, + "mae": 0.34554752707481384, + "r2": 0.5472283363342285, + "score": -0.4439757847261177 + }, + "val": { + "rmse": 0.4634085546239816, + "mae": 0.36031344532966614, + "r2": 0.5075141191482544, + "score": -0.4634085546239816 + }, + "test": { + "rmse": 0.4808915146871742, + "mae": 0.37534573674201965, + "r2": 0.4843977689743042, + "score": -0.4808915146871742 + } + }, + "time": "0:12:07.675037", + "diversity": { + "train": { + "mean_centered_corr": -0.019306395202875137, + "mean_pairwise_disagreement": NaN, + "member_std": 0.057879555970430374 + }, + "val": { + "mean_centered_corr": -0.01832164078950882, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06112360581755638 + }, + "test": { + "mean_centered_corr": -0.01794510707259178, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06266573071479797 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-evaluation/1.toml new file mode 100644 index 000000000..b98101f71 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-evaluation/1.toml @@ -0,0 +1,46 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 1 +head_selection = true + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.8 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 1 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-evaluation/1/report.json new file mode 100644 index 000000000..e4dab74b3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-evaluation/1/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + }, + "mask_stats": { + "keep_rate_actual": 0.8004557490348816, + "min_features_per_member": 142.0, + "mean_features_per_member": 153.6875, + "max_features_per_member": 165.0, + "min_members_per_feature": 19.0, + "mean_members_per_feature": 25.61458396911621, + "max_members_per_feature": 30.0, + "mean_pairwise_jaccard": 0.6664211145724647, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 18870, + "metrics": { + "train": { + "rmse": 0.4457924916226811, + "mae": 0.3470214605331421, + "r2": 0.5435153841972351, + "score": -0.4457924916226811 + }, + "val": { + "rmse": 0.4632980549483924, + "mae": 0.3602248728275299, + "r2": 0.5077489018440247, + "score": -0.4632980549483924 + }, + "test": { + "rmse": 0.48077915988352504, + "mae": 0.3752482533454895, + "r2": 0.48463869094848633, + "score": -0.48077915988352504 + } + }, + "time": "0:10:14.470810", + "diversity": { + "train": { + "mean_centered_corr": -0.019374029710888863, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05610343813896179 + }, + "val": { + "mean_centered_corr": -0.018558263778686523, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05900688096880913 + }, + "test": { + "mean_centered_corr": -0.018259482458233833, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06036880984902382 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-evaluation/2.toml new file mode 100644 index 000000000..308b77981 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-evaluation/2.toml @@ -0,0 +1,46 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 2 +head_selection = true + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.8 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 2 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-evaluation/2/report.json new file mode 100644 index 000000000..c577c12b6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-evaluation/2/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + }, + "mask_stats": { + "keep_rate_actual": 0.8037109375, + "min_features_per_member": 144.0, + "mean_features_per_member": 154.3125, + "max_features_per_member": 166.0, + "min_members_per_feature": 19.0, + "mean_members_per_feature": 25.71875, + "max_members_per_feature": 31.0, + "mean_pairwise_jaccard": 0.6710767481400324, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 28638, + "metrics": { + "train": { + "rmse": 0.44161565421565047, + "mae": 0.34365126490592957, + "r2": 0.5520293712615967, + "score": -0.44161565421565047 + }, + "val": { + "rmse": 0.4630663413507467, + "mae": 0.35984113812446594, + "r2": 0.5082411766052246, + "score": -0.4630663413507467 + }, + "test": { + "rmse": 0.48078120546833536, + "mae": 0.3750513195991516, + "r2": 0.48463428020477295, + "score": -0.48078120546833536 + } + }, + "time": "0:14:35.891802", + "diversity": { + "train": { + "mean_centered_corr": -0.019855137914419174, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06328506022691727 + }, + "val": { + "mean_centered_corr": -0.01927705854177475, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06649371236562729 + }, + "test": { + "mean_centered_corr": -0.018950071185827255, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06782469898462296 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..c9f5dc71a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,193 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.8019205927848816, + "min_features_per_member": 141.0, + "mean_features_per_member": 153.96875, + "max_features_per_member": 167.0, + "min_members_per_feature": 21.0, + "mean_members_per_feature": 25.66145896911621, + "max_members_per_feature": 31.0, + "mean_pairwise_jaccard": 0.6682771134235742, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44390037903332324, + "mae": 0.34550046920776367, + "r2": 0.547382116317749, + "score": -0.44390037903332324 + }, + "val": { + "rmse": 0.4624822561623469, + "mae": 0.35955899953842163, + "r2": 0.5094809532165527, + "score": -0.4624822561623469 + }, + "test": { + "rmse": 0.4798057758257153, + "mae": 0.37443193793296814, + "r2": 0.4867233633995056, + "score": -0.4798057758257153 + } + }, + "time": "0:12:07.675037", + "diversity": { + "train": { + "mean_centered_corr": -0.019306395202875137, + "mean_pairwise_disagreement": NaN, + "member_std": 0.057879555970430374 + }, + "val": { + "mean_centered_corr": -0.01832164078950882, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06112360581755638 + }, + "test": { + "mean_centered_corr": -0.01794510707259178, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06266573071479797 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 19, + 27, + 17, + 24, + 25, + 20, + 22, + 4, + 14, + 7, + 5 + ], + "head_selection_time": "0:00:00.407380" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/1/report.json new file mode 100644 index 000000000..84931fdf6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/1/report.json @@ -0,0 +1,202 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + }, + "mask_stats": { + "keep_rate_actual": 0.8004557490348816, + "min_features_per_member": 142.0, + "mean_features_per_member": 153.6875, + "max_features_per_member": 165.0, + "min_members_per_feature": 19.0, + "mean_members_per_feature": 25.61458396911621, + "max_members_per_feature": 30.0, + "mean_pairwise_jaccard": 0.6664211145724647, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 18870, + "metrics": { + "train": { + "rmse": 0.44578164467993003, + "mae": 0.34703874588012695, + "r2": 0.5435376167297363, + "score": -0.44578164467993003 + }, + "val": { + "rmse": 0.4626697544496206, + "mae": 0.3596174120903015, + "r2": 0.5090831518173218, + "score": -0.4626697544496206 + }, + "test": { + "rmse": 0.47992630708272743, + "mae": 0.37454068660736084, + "r2": 0.4864654541015625, + "score": -0.47992630708272743 + } + }, + "time": "0:10:14.470810", + "diversity": { + "train": { + "mean_centered_corr": -0.019374029710888863, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05610343813896179 + }, + "val": { + "mean_centered_corr": -0.018558263778686523, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05900688096880913 + }, + "test": { + "mean_centered_corr": -0.018259482458233833, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06036880984902382 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 27, + 10, + 18, + 4, + 2, + 23, + 28, + 6, + 24, + 11, + 22, + 20, + 25, + 10, + 19, + 30, + 28, + 18, + 15, + 9 + ], + "head_selection_time": "0:00:00.867913" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/2/report.json new file mode 100644 index 000000000..d99ab4966 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/2/report.json @@ -0,0 +1,193 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + }, + "mask_stats": { + "keep_rate_actual": 0.8037109375, + "min_features_per_member": 144.0, + "mean_features_per_member": 154.3125, + "max_features_per_member": 166.0, + "min_members_per_feature": 19.0, + "mean_members_per_feature": 25.71875, + "max_members_per_feature": 31.0, + "mean_pairwise_jaccard": 0.6710767481400324, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 28638, + "metrics": { + "train": { + "rmse": 0.44202025927421124, + "mae": 0.34406524896621704, + "r2": 0.5512081384658813, + "score": -0.44202025927421124 + }, + "val": { + "rmse": 0.462357531977379, + "mae": 0.3593025505542755, + "r2": 0.5097454786300659, + "score": -0.462357531977379 + }, + "test": { + "rmse": 0.48018644149208806, + "mae": 0.37463948130607605, + "r2": 0.485908567905426, + "score": -0.48018644149208806 + } + }, + "time": "0:14:35.891802", + "diversity": { + "train": { + "mean_centered_corr": -0.019855137914419174, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06328506022691727 + }, + "val": { + "mean_centered_corr": -0.01927705854177475, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06649371236562729 + }, + "test": { + "mean_centered_corr": -0.018950071185827255, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06782469898462296 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 19, + 13, + 29, + 15, + 7, + 8, + 24, + 13, + 17, + 26, + 30 + ], + "head_selection_time": "0:00:00.396628" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..8abb38b47 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-best-head-evaluation/0/report.json @@ -0,0 +1,156 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4475775717786638, + "mae": 0.3483501672744751, + "r2": 0.5398522615432739, + "score": -0.4475775717786638 + }, + "val": { + "rmse": 0.4649795846661889, + "mae": 0.361278235912323, + "r2": 0.504169225692749, + "score": -0.4649795846661889 + }, + "test": { + "rmse": 0.4827695666979872, + "mae": 0.3767538070678711, + "r2": 0.4803627133369446, + "score": -0.4827695666979872 + } + }, + "time": "0:23:06.357271", + "diversity": { + "train": { + "mean_centered_corr": -0.02037157490849495, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0513317808508873 + }, + "val": { + "mean_centered_corr": -0.020479049533605576, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05293743312358856 + }, + "test": { + "mean_centered_corr": -0.02046302892267704, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05355745181441307 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 7 + ], + "head_selection_time": "0:00:00.028943" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-best-head-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-best-head-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-best-head-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-best-head-evaluation/1/report.json new file mode 100644 index 000000000..4e96471bb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-best-head-evaluation/1/report.json @@ -0,0 +1,156 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 19092, + "metrics": { + "train": { + "rmse": 0.4483249230582749, + "mae": 0.34914642572402954, + "r2": 0.5383143424987793, + "score": -0.4483249230582749 + }, + "val": { + "rmse": 0.46454139300633795, + "mae": 0.3614574670791626, + "r2": 0.5051032900810242, + "score": -0.46454139300633795 + }, + "test": { + "rmse": 0.4826743974666991, + "mae": 0.37691187858581543, + "r2": 0.4805675745010376, + "score": -0.4826743974666991 + } + }, + "time": "0:19:42.335963", + "diversity": { + "train": { + "mean_centered_corr": -0.020455842837691307, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048325903713703156 + }, + "val": { + "mean_centered_corr": -0.020279740914702415, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04973817616701126 + }, + "test": { + "mean_centered_corr": -0.02030608057975769, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05035385116934776 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 5 + ], + "head_selection_time": "0:00:00.028060" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-best-head-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-best-head-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-best-head-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-best-head-evaluation/2/report.json new file mode 100644 index 000000000..c1662c3e5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-best-head-evaluation/2/report.json @@ -0,0 +1,156 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 28638, + "metrics": { + "train": { + "rmse": 0.44586212972829725, + "mae": 0.34740886092185974, + "r2": 0.5433727502822876, + "score": -0.44586212972829725 + }, + "val": { + "rmse": 0.46551838172590837, + "mae": 0.3620719611644745, + "r2": 0.5030194520950317, + "score": -0.46551838172590837 + }, + "test": { + "rmse": 0.4845241655350321, + "mae": 0.3784400224685669, + "r2": 0.47657865285873413, + "score": -0.4845241655350321 + } + }, + "time": "0:27:52.069713", + "diversity": { + "train": { + "mean_centered_corr": -0.020423956215381622, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05755901709198952 + }, + "val": { + "mean_centered_corr": -0.020522931590676308, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05942666903138161 + }, + "test": { + "mean_centered_corr": -0.02022053673863411, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05996261537075043 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 25 + ], + "head_selection_time": "0:00:00.027846" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-evaluation/0.toml new file mode 100644 index 000000000..364d0e3c9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-evaluation/0/report.json new file mode 100644 index 000000000..35053ecb2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-evaluation/0/report.json @@ -0,0 +1,152 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4435449287553804, + "mae": 0.3451736271381378, + "r2": 0.5481066703796387, + "score": -0.4435449287553804 + }, + "val": { + "rmse": 0.46272482498106277, + "mae": 0.3595674932003021, + "r2": 0.5089662671089172, + "score": -0.46272482498106277 + }, + "test": { + "rmse": 0.4802920784266626, + "mae": 0.37473270297050476, + "r2": 0.4856823682785034, + "score": -0.4802920784266626 + } + }, + "time": "0:23:06.357271", + "diversity": { + "train": { + "mean_centered_corr": -0.02037157490849495, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0513317808508873 + }, + "val": { + "mean_centered_corr": -0.020479049533605576, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05293743312358856 + }, + "test": { + "mean_centered_corr": -0.02046302892267704, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05355745181441307 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-evaluation/1.toml new file mode 100644 index 000000000..647925031 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-evaluation/1.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 1 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-evaluation/1/report.json new file mode 100644 index 000000000..43edb6098 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-evaluation/1/report.json @@ -0,0 +1,152 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 19092, + "metrics": { + "train": { + "rmse": 0.4453989580783725, + "mae": 0.34662750363349915, + "r2": 0.5443209409713745, + "score": -0.4453989580783725 + }, + "val": { + "rmse": 0.4627833825636106, + "mae": 0.35961589217185974, + "r2": 0.5088419914245605, + "score": -0.4627833825636106 + }, + "test": { + "rmse": 0.4804816670736117, + "mae": 0.3748900294303894, + "r2": 0.4852762818336487, + "score": -0.4804816670736117 + } + }, + "time": "0:19:42.335963", + "diversity": { + "train": { + "mean_centered_corr": -0.020455842837691307, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048325903713703156 + }, + "val": { + "mean_centered_corr": -0.020279740914702415, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04973817616701126 + }, + "test": { + "mean_centered_corr": -0.02030608057975769, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05035385116934776 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-evaluation/2.toml new file mode 100644 index 000000000..6c85aa21c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-evaluation/2.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 2 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-evaluation/2/report.json new file mode 100644 index 000000000..605a6aa11 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-evaluation/2/report.json @@ -0,0 +1,152 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 28638, + "metrics": { + "train": { + "rmse": 0.44116400088657864, + "mae": 0.3432346284389496, + "r2": 0.5529451370239258, + "score": -0.44116400088657864 + }, + "val": { + "rmse": 0.4626829107835815, + "mae": 0.35933375358581543, + "r2": 0.5090552568435669, + "score": -0.4626829107835815 + }, + "test": { + "rmse": 0.4805534877331418, + "mae": 0.37477874755859375, + "r2": 0.4851223826408386, + "score": -0.4805534877331418 + } + }, + "time": "0:27:52.069713", + "diversity": { + "train": { + "mean_centered_corr": -0.020423956215381622, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05755901709198952 + }, + "val": { + "mean_centered_corr": -0.020522931590676308, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05942666903138161 + }, + "test": { + "mean_centered_corr": -0.02022053673863411, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05996261537075043 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..298a7858c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,178 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44362437525125054, + "mae": 0.3453079164028168, + "r2": 0.5479447841644287, + "score": -0.44362437525125054 + }, + "val": { + "rmse": 0.4624107384246987, + "mae": 0.35940805077552795, + "r2": 0.5096326470375061, + "score": -0.4624107384246987 + }, + "test": { + "rmse": 0.479974942646056, + "mae": 0.37463974952697754, + "r2": 0.48636138439178467, + "score": -0.479974942646056 + } + }, + "time": "0:23:06.357271", + "diversity": { + "train": { + "mean_centered_corr": -0.02037157490849495, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0513317808508873 + }, + "val": { + "mean_centered_corr": -0.020479049533605576, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05293743312358856 + }, + "test": { + "mean_centered_corr": -0.02046302892267704, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05355745181441307 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 7, + 31, + 14, + 17, + 3, + 4, + 5, + 24, + 27, + 22, + 10, + 6, + 14, + 31, + 17, + 3, + 5, + 16, + 24, + 4, + 9, + 27, + 14 + ], + "head_selection_time": "0:00:01.063797" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/1/report.json new file mode 100644 index 000000000..d7097eada --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/1/report.json @@ -0,0 +1,169 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 19092, + "metrics": { + "train": { + "rmse": 0.4455662222054085, + "mae": 0.34683141112327576, + "r2": 0.5439786911010742, + "score": -0.4455662222054085 + }, + "val": { + "rmse": 0.46252858614858405, + "mae": 0.359448105096817, + "r2": 0.5093826651573181, + "score": -0.46252858614858405 + }, + "test": { + "rmse": 0.48029415711103896, + "mae": 0.37478509545326233, + "r2": 0.48567795753479004, + "score": -0.48029415711103896 + } + }, + "time": "0:19:42.335963", + "diversity": { + "train": { + "mean_centered_corr": -0.020455842837691307, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048325903713703156 + }, + "val": { + "mean_centered_corr": -0.020279740914702415, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04973817616701126 + }, + "test": { + "mean_centered_corr": -0.02030608057975769, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05035385116934776 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 5, + 25, + 15, + 13, + 8, + 30, + 24, + 6, + 9, + 4, + 25, + 15, + 13, + 27 + ], + "head_selection_time": "0:00:00.544937" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/2/report.json new file mode 100644 index 000000000..fbb58df48 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/2/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 28638, + "metrics": { + "train": { + "rmse": 0.4413632397247652, + "mae": 0.3435739278793335, + "r2": 0.5525412559509277, + "score": -0.4413632397247652 + }, + "val": { + "rmse": 0.462385779530238, + "mae": 0.3592732548713684, + "r2": 0.5096855759620667, + "score": -0.462385779530238 + }, + "test": { + "rmse": 0.4801696683856488, + "mae": 0.3746996223926544, + "r2": 0.4859445095062256, + "score": -0.4801696683856488 + } + }, + "time": "0:27:52.069713", + "diversity": { + "train": { + "mean_centered_corr": -0.020423956215381622, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05755901709198952 + }, + "val": { + "mean_centered_corr": -0.020522931590676308, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05942666903138161 + }, + "test": { + "mean_centered_corr": -0.02022053673863411, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05996261537075043 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 25, + 13, + 21, + 19, + 6, + 7, + 1, + 31, + 15, + 23, + 27, + 30, + 13, + 14, + 5, + 19, + 21, + 22, + 13, + 29, + 6, + 7, + 2, + 31 + ], + "head_selection_time": "0:00:01.133563" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-best-head-evaluation/0/report.json new file mode 100644 index 000000000..55ac8b264 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-best-head-evaluation/0/report.json @@ -0,0 +1,168 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5112224822345067, + "mae": 0.40589213371276855, + "r2": 0.5101465582847595, + "score": -0.5112224822345067 + }, + "val": { + "rmse": 0.5583881453279098, + "mae": 0.4439501166343689, + "r2": 0.412036657333374, + "score": -0.5583881453279098 + }, + "test": { + "rmse": 0.5557654209580059, + "mae": 0.4505227506160736, + "r2": 0.42443811893463135, + "score": -0.5557654209580059 + } + }, + "time": "0:07:16.600998", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023548" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-best-head-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-best-head-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-best-head-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-best-head-evaluation/1/report.json new file mode 100644 index 000000000..146ca13ec --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-best-head-evaluation/1/report.json @@ -0,0 +1,168 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5157048423679549, + "mae": 0.408806711435318, + "r2": 0.5015189051628113, + "score": -0.5157048423679549 + }, + "val": { + "rmse": 0.5578575885405295, + "mae": 0.44588109850883484, + "r2": 0.41315340995788574, + "score": -0.5578575885405295 + }, + "test": { + "rmse": 0.5568376961917461, + "mae": 0.4522704780101776, + "r2": 0.4222150444984436, + "score": -0.5568376961917461 + } + }, + "time": "0:06:43.806101", + "diversity": { + "train": { + "mean_centered_corr": -0.02029341645538807, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06168676167726517 + }, + "val": { + "mean_centered_corr": -0.020232515409588814, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0631292313337326 + }, + "test": { + "mean_centered_corr": -0.020325176417827606, + "mean_pairwise_disagreement": NaN, + "member_std": 0.061680618673563004 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 22 + ], + "head_selection_time": "0:00:00.023481" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-best-head-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-best-head-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-best-head-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-best-head-evaluation/2/report.json new file mode 100644 index 000000000..708292a23 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-best-head-evaluation/2/report.json @@ -0,0 +1,168 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5148308736438392, + "mae": 0.40706372261047363, + "r2": 0.5032070279121399, + "score": -0.5148308736438392 + }, + "val": { + "rmse": 0.5576450053690213, + "mae": 0.4445996880531311, + "r2": 0.4136006236076355, + "score": -0.5576450053690213 + }, + "test": { + "rmse": 0.5574656479348947, + "mae": 0.45258787274360657, + "r2": 0.42091113328933716, + "score": -0.5574656479348947 + } + }, + "time": "0:06:38.793392", + "diversity": { + "train": { + "mean_centered_corr": -0.0205259807407856, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06119334325194359 + }, + "val": { + "mean_centered_corr": -0.020698808133602142, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06522355228662491 + }, + "test": { + "mean_centered_corr": -0.020667163655161858, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06273415684700012 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 8 + ], + "head_selection_time": "0:00:00.023044" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-evaluation/0.toml new file mode 100644 index 000000000..d07370fc8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-evaluation/0.toml @@ -0,0 +1,33 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-evaluation/0/report.json new file mode 100644 index 000000000..5bbb360c8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-evaluation/0/report.json @@ -0,0 +1,164 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5040057771555991, + "mae": 0.39488768577575684, + "r2": 0.5238790512084961, + "score": -0.5040057771555991 + }, + "val": { + "rmse": 0.5558749906446844, + "mae": 0.4398687481880188, + "r2": 0.4173172116279602, + "score": -0.5558749906446844 + }, + "test": { + "rmse": 0.5496162918403876, + "mae": 0.44109272956848145, + "r2": 0.4371039867401123, + "score": -0.5496162918403876 + } + }, + "time": "0:07:16.600998", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-evaluation/1.toml new file mode 100644 index 000000000..0e316c4c0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-evaluation/1.toml @@ -0,0 +1,33 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 1 +head_selection = true + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-evaluation/1/report.json new file mode 100644 index 000000000..2147e5dc2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-evaluation/1/report.json @@ -0,0 +1,164 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5105330146478386, + "mae": 0.40035760402679443, + "r2": 0.5114669799804688, + "score": -0.5105330146478386 + }, + "val": { + "rmse": 0.5562247345576334, + "mae": 0.439607709646225, + "r2": 0.416583776473999, + "score": -0.5562247345576334 + }, + "test": { + "rmse": 0.550284872190118, + "mae": 0.44156232476234436, + "r2": 0.4357336759567261, + "score": -0.550284872190118 + } + }, + "time": "0:06:43.806101", + "diversity": { + "train": { + "mean_centered_corr": -0.02029341645538807, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06168676167726517 + }, + "val": { + "mean_centered_corr": -0.020232515409588814, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0631292313337326 + }, + "test": { + "mean_centered_corr": -0.020325176417827606, + "mean_pairwise_disagreement": NaN, + "member_std": 0.061680618673563004 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-evaluation/2.toml new file mode 100644 index 000000000..5d2c3b3bb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-evaluation/2.toml @@ -0,0 +1,33 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 2 +head_selection = true + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-evaluation/2/report.json new file mode 100644 index 000000000..2b292662d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-evaluation/2/report.json @@ -0,0 +1,164 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5116329266744724, + "mae": 0.4038048982620239, + "r2": 0.5093597173690796, + "score": -0.5116329266744724 + }, + "val": { + "rmse": 0.5557493871736087, + "mae": 0.4417022466659546, + "r2": 0.4175804853439331, + "score": -0.5557493871736087 + }, + "test": { + "rmse": 0.5507763801521746, + "mae": 0.44514498114585876, + "r2": 0.43472522497177124, + "score": -0.5507763801521746 + } + }, + "time": "0:06:38.793392", + "diversity": { + "train": { + "mean_centered_corr": -0.0205259807407856, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06119334325194359 + }, + "val": { + "mean_centered_corr": -0.020698808133602142, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06522355228662491 + }, + "test": { + "mean_centered_corr": -0.020667163655161858, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06273415684700012 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..fc04b59cb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/0/report.json @@ -0,0 +1,181 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5042286809765437, + "mae": 0.39605751633644104, + "r2": 0.5234577655792236, + "score": -0.5042286809765437 + }, + "val": { + "rmse": 0.5553348954063525, + "mae": 0.44016024470329285, + "r2": 0.4184489846229553, + "score": -0.5553348954063525 + }, + "test": { + "rmse": 0.550360417400049, + "mae": 0.44305530190467834, + "r2": 0.43557870388031006, + "score": -0.550360417400049 + } + }, + "time": "0:07:16.600998", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13, + 6, + 10, + 14, + 26, + 27, + 18, + 25, + 6, + 15, + 8, + 17, + 10, + 26 + ], + "head_selection_time": "0:00:00.432507" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/1/report.json new file mode 100644 index 000000000..ecd089b66 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/1/report.json @@ -0,0 +1,176 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5108096659852551, + "mae": 0.4014206528663635, + "r2": 0.5109373331069946, + "score": -0.5108096659852551 + }, + "val": { + "rmse": 0.5553836483690917, + "mae": 0.440256267786026, + "r2": 0.4183468818664551, + "score": -0.5553836483690917 + }, + "test": { + "rmse": 0.5515298017800222, + "mae": 0.4441634714603424, + "r2": 0.433177649974823, + "score": -0.5515298017800222 + } + }, + "time": "0:06:43.806101", + "diversity": { + "train": { + "mean_centered_corr": -0.02029341645538807, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06168676167726517 + }, + "val": { + "mean_centered_corr": -0.020232515409588814, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0631292313337326 + }, + "test": { + "mean_centered_corr": -0.020325176417827606, + "mean_pairwise_disagreement": NaN, + "member_std": 0.061680618673563004 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 22, + 30, + 29, + 19, + 11, + 18, + 30, + 4, + 16 + ], + "head_selection_time": "0:00:00.255273" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/2/report.json new file mode 100644 index 000000000..5cc033c90 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/2/report.json @@ -0,0 +1,176 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5120909698461144, + "mae": 0.4043644666671753, + "r2": 0.5084807872772217, + "score": -0.5120909698461144 + }, + "val": { + "rmse": 0.5554437183690072, + "mae": 0.4415339529514313, + "r2": 0.41822099685668945, + "score": -0.5554437183690072 + }, + "test": { + "rmse": 0.5516331354574737, + "mae": 0.44613465666770935, + "r2": 0.4329652190208435, + "score": -0.5516331354574737 + } + }, + "time": "0:06:38.793392", + "diversity": { + "train": { + "mean_centered_corr": -0.0205259807407856, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06119334325194359 + }, + "val": { + "mean_centered_corr": -0.020698808133602142, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06522355228662491 + }, + "test": { + "mean_centered_corr": -0.020667163655161858, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06273415684700012 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 8, + 31, + 22, + 14, + 12, + 0, + 16, + 27, + 13 + ], + "head_selection_time": "0:00:00.254917" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..dd789361b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-best-head-evaluation/0/report.json @@ -0,0 +1,622 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5112224822345067, + "mae": 0.40589213371276855, + "r2": 0.5101465582847595, + "score": -0.5112224822345067 + }, + "val": { + "rmse": 0.5583881453279098, + "mae": 0.4439501166343689, + "r2": 0.412036657333374, + "score": -0.5583881453279098 + }, + "test": { + "rmse": 0.5557654209580059, + "mae": 0.4505227506160736, + "r2": 0.42443811893463135, + "score": -0.5557654209580059 + } + }, + "time": "0:07:43.023818", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023628" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-best-head-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-best-head-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-best-head-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-best-head-evaluation/1/report.json new file mode 100644 index 000000000..5dd192787 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-best-head-evaluation/1/report.json @@ -0,0 +1,622 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5157048423679549, + "mae": 0.408806711435318, + "r2": 0.5015189051628113, + "score": -0.5157048423679549 + }, + "val": { + "rmse": 0.5578575885405295, + "mae": 0.44588109850883484, + "r2": 0.41315340995788574, + "score": -0.5578575885405295 + }, + "test": { + "rmse": 0.5568376961917461, + "mae": 0.4522704780101776, + "r2": 0.4222150444984436, + "score": -0.5568376961917461 + } + }, + "time": "0:07:06.135890", + "diversity": { + "train": { + "mean_centered_corr": -0.02029341645538807, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06168676167726517 + }, + "val": { + "mean_centered_corr": -0.020232515409588814, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0631292313337326 + }, + "test": { + "mean_centered_corr": -0.020325176417827606, + "mean_pairwise_disagreement": NaN, + "member_std": 0.061680618673563004 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 22 + ], + "head_selection_time": "0:00:00.023866" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-best-head-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-best-head-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-best-head-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-best-head-evaluation/2/report.json new file mode 100644 index 000000000..ac1798ed1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-best-head-evaluation/2/report.json @@ -0,0 +1,622 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5148308736438392, + "mae": 0.40706372261047363, + "r2": 0.5032070279121399, + "score": -0.5148308736438392 + }, + "val": { + "rmse": 0.5576450053690213, + "mae": 0.4445996880531311, + "r2": 0.4136006236076355, + "score": -0.5576450053690213 + }, + "test": { + "rmse": 0.5574656479348947, + "mae": 0.45258787274360657, + "r2": 0.42091113328933716, + "score": -0.5574656479348947 + } + }, + "time": "0:07:04.841597", + "diversity": { + "train": { + "mean_centered_corr": -0.0205259807407856, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06119334325194359 + }, + "val": { + "mean_centered_corr": -0.020698808133602142, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06522355228662491 + }, + "test": { + "mean_centered_corr": -0.020667163655161858, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06273415684700012 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 8 + ], + "head_selection_time": "0:00:00.023514" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/0.toml new file mode 100644 index 000000000..e7d7fb4f3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/0.toml @@ -0,0 +1,151 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "delivery-eta" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] +start_epoch = 16 +start_fraction = 0.5 +mode = "consensus_raw" + +[cf_fisd.member_groups] +xgb = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] +lgbm = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] +cat = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/0/report.json new file mode 100644 index 000000000..c88d18129 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/0/report.json @@ -0,0 +1,618 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5040057771555991, + "mae": 0.39488768577575684, + "r2": 0.5238790512084961, + "score": -0.5040057771555991 + }, + "val": { + "rmse": 0.5558749906446844, + "mae": 0.4398687481880188, + "r2": 0.4173172116279602, + "score": -0.5558749906446844 + }, + "test": { + "rmse": 0.5496162918403876, + "mae": 0.44109272956848145, + "r2": 0.4371039867401123, + "score": -0.5496162918403876 + } + }, + "time": "0:07:43.023818", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/1.toml new file mode 100644 index 000000000..59de1c3ce --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/1.toml @@ -0,0 +1,151 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 1 +head_selection = true + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "delivery-eta" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] +start_epoch = 16 +start_fraction = 0.5 +mode = "consensus_raw" + +[cf_fisd.member_groups] +xgb = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] +lgbm = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] +cat = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/1/report.json new file mode 100644 index 000000000..c6460e960 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/1/report.json @@ -0,0 +1,618 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5105330146478386, + "mae": 0.40035760402679443, + "r2": 0.5114669799804688, + "score": -0.5105330146478386 + }, + "val": { + "rmse": 0.5562247345576334, + "mae": 0.439607709646225, + "r2": 0.416583776473999, + "score": -0.5562247345576334 + }, + "test": { + "rmse": 0.550284872190118, + "mae": 0.44156232476234436, + "r2": 0.4357336759567261, + "score": -0.550284872190118 + } + }, + "time": "0:07:06.135890", + "diversity": { + "train": { + "mean_centered_corr": -0.02029341645538807, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06168676167726517 + }, + "val": { + "mean_centered_corr": -0.020232515409588814, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0631292313337326 + }, + "test": { + "mean_centered_corr": -0.020325176417827606, + "mean_pairwise_disagreement": NaN, + "member_std": 0.061680618673563004 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/2.toml new file mode 100644 index 000000000..25ebc2e28 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/2.toml @@ -0,0 +1,151 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 2 +head_selection = true + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "delivery-eta" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] +start_epoch = 16 +start_fraction = 0.5 +mode = "consensus_raw" + +[cf_fisd.member_groups] +xgb = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] +lgbm = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] +cat = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/2/report.json new file mode 100644 index 000000000..a78e05ecf --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/2/report.json @@ -0,0 +1,618 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5116329266744724, + "mae": 0.4038048982620239, + "r2": 0.5093597173690796, + "score": -0.5116329266744724 + }, + "val": { + "rmse": 0.5557493871736087, + "mae": 0.4417022466659546, + "r2": 0.4175804853439331, + "score": -0.5557493871736087 + }, + "test": { + "rmse": 0.5507763801521746, + "mae": 0.44514498114585876, + "r2": 0.43472522497177124, + "score": -0.5507763801521746 + } + }, + "time": "0:07:04.841597", + "diversity": { + "train": { + "mean_centered_corr": -0.0205259807407856, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06119334325194359 + }, + "val": { + "mean_centered_corr": -0.020698808133602142, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06522355228662491 + }, + "test": { + "mean_centered_corr": -0.020667163655161858, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06273415684700012 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..da3358e87 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,635 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5042286809765437, + "mae": 0.39605751633644104, + "r2": 0.5234577655792236, + "score": -0.5042286809765437 + }, + "val": { + "rmse": 0.5553348954063525, + "mae": 0.44016024470329285, + "r2": 0.4184489846229553, + "score": -0.5553348954063525 + }, + "test": { + "rmse": 0.550360417400049, + "mae": 0.44305530190467834, + "r2": 0.43557870388031006, + "score": -0.550360417400049 + } + }, + "time": "0:07:43.023818", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13, + 6, + 10, + 14, + 26, + 27, + 18, + 25, + 6, + 15, + 8, + 17, + 10, + 26 + ], + "head_selection_time": "0:00:00.432213" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/1/report.json new file mode 100644 index 000000000..24cd05329 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/1/report.json @@ -0,0 +1,630 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5108096659852551, + "mae": 0.4014206528663635, + "r2": 0.5109373331069946, + "score": -0.5108096659852551 + }, + "val": { + "rmse": 0.5553836483690917, + "mae": 0.440256267786026, + "r2": 0.4183468818664551, + "score": -0.5553836483690917 + }, + "test": { + "rmse": 0.5515298017800222, + "mae": 0.4441634714603424, + "r2": 0.433177649974823, + "score": -0.5515298017800222 + } + }, + "time": "0:07:06.135890", + "diversity": { + "train": { + "mean_centered_corr": -0.02029341645538807, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06168676167726517 + }, + "val": { + "mean_centered_corr": -0.020232515409588814, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0631292313337326 + }, + "test": { + "mean_centered_corr": -0.020325176417827606, + "mean_pairwise_disagreement": NaN, + "member_std": 0.061680618673563004 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 22, + 30, + 29, + 19, + 11, + 18, + 30, + 4, + 16 + ], + "head_selection_time": "0:00:00.266709" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/2/report.json new file mode 100644 index 000000000..29d133635 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/2/report.json @@ -0,0 +1,630 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5120909698461144, + "mae": 0.4043644666671753, + "r2": 0.5084807872772217, + "score": -0.5120909698461144 + }, + "val": { + "rmse": 0.5554437183690072, + "mae": 0.4415339529514313, + "r2": 0.41822099685668945, + "score": -0.5554437183690072 + }, + "test": { + "rmse": 0.5516331354574737, + "mae": 0.44613465666770935, + "r2": 0.4329652190208435, + "score": -0.5516331354574737 + } + }, + "time": "0:07:04.841597", + "diversity": { + "train": { + "mean_centered_corr": -0.0205259807407856, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06119334325194359 + }, + "val": { + "mean_centered_corr": -0.020698808133602142, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06522355228662491 + }, + "test": { + "mean_centered_corr": -0.020667163655161858, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06273415684700012 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 8, + 31, + 22, + 14, + 12, + 0, + 16, + 27, + 13 + ], + "head_selection_time": "0:00:00.259040" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/0/report.json new file mode 100644 index 000000000..759452322 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/0/report.json @@ -0,0 +1,633 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5175918577279733, + "mae": 0.40775081515312195, + "r2": 0.4978642463684082, + "score": -0.5175918577279733 + }, + "val": { + "rmse": 0.5572652691682626, + "mae": 0.44076380133628845, + "r2": 0.4143989086151123, + "score": -0.5572652691682626 + }, + "test": { + "rmse": 0.5524207799310179, + "mae": 0.4452172815799713, + "r2": 0.4313448667526245, + "score": -0.5524207799310179 + } + }, + "time": "0:15:04.980270", + "diversity": { + "train": { + "mean_centered_corr": -0.02016017585992813, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06105273962020874 + }, + "val": { + "mean_centered_corr": -0.01993357017636299, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06221357360482216 + }, + "test": { + "mean_centered_corr": -0.020221160724759102, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06099524348974228 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 11 + ], + "head_selection_time": "0:00:00.023383" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/1/report.json new file mode 100644 index 000000000..01eb91f6a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/1/report.json @@ -0,0 +1,633 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5171067304438404, + "mae": 0.40677744150161743, + "r2": 0.49880504608154297, + "score": -0.5171067304438404 + }, + "val": { + "rmse": 0.5577909128836495, + "mae": 0.44260263442993164, + "r2": 0.41329365968704224, + "score": -0.5577909128836495 + }, + "test": { + "rmse": 0.5532065757037241, + "mae": 0.44593191146850586, + "r2": 0.4297259449958801, + "score": -0.5532065757037241 + } + }, + "time": "0:15:04.310558", + "diversity": { + "train": { + "mean_centered_corr": -0.020146815106272697, + "mean_pairwise_disagreement": NaN, + "member_std": 0.061014287173748016 + }, + "val": { + "mean_centered_corr": -0.020077988505363464, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06329856067895889 + }, + "test": { + "mean_centered_corr": -0.01972520351409912, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06241019815206528 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 5 + ], + "head_selection_time": "0:00:00.023906" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/2/report.json new file mode 100644 index 000000000..e40774ea4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/2/report.json @@ -0,0 +1,633 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1911, + "metrics": { + "train": { + "rmse": 0.5137978750972679, + "mae": 0.4069885015487671, + "r2": 0.50519859790802, + "score": -0.5137978750972679 + }, + "val": { + "rmse": 0.5585470113340857, + "mae": 0.44582831859588623, + "r2": 0.4117019772529602, + "score": -0.5585470113340857 + }, + "test": { + "rmse": 0.5573737419749005, + "mae": 0.4522700607776642, + "r2": 0.4211021065711975, + "score": -0.5573737419749005 + } + }, + "time": "0:16:37.086669", + "diversity": { + "train": { + "mean_centered_corr": -0.020242199301719666, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0667627602815628 + }, + "val": { + "mean_centered_corr": -0.02033313177525997, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06825815886259079 + }, + "test": { + "mean_centered_corr": -0.020218119025230408, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06682927906513214 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 5 + ], + "head_selection_time": "0:00:00.023427" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/0.toml new file mode 100644 index 000000000..a8cfa8c75 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/0.toml @@ -0,0 +1,158 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.5 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "delivery-eta" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] +start_epoch = 16 +start_fraction = 0.5 +mode = "consensus_raw" + +[cf_fisd.member_groups] +xgb = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] +lgbm = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] +cat = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/0/report.json new file mode 100644 index 000000000..49ee599d9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/0/report.json @@ -0,0 +1,629 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5150130136938161, + "mae": 0.40628108382225037, + "r2": 0.5028554797172546, + "score": -0.5150130136938161 + }, + "val": { + "rmse": 0.5559356776481553, + "mae": 0.441616415977478, + "r2": 0.4171900153160095, + "score": -0.5559356776481553 + }, + "test": { + "rmse": 0.55135486033622, + "mae": 0.445721298456192, + "r2": 0.43353718519210815, + "score": -0.55135486033622 + } + }, + "time": "0:15:04.980270", + "diversity": { + "train": { + "mean_centered_corr": -0.02016017585992813, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06105273962020874 + }, + "val": { + "mean_centered_corr": -0.01993357017636299, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06221357360482216 + }, + "test": { + "mean_centered_corr": -0.020221160724759102, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06099524348974228 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/1.toml new file mode 100644 index 000000000..7c4ec352c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/1.toml @@ -0,0 +1,158 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 1 +head_selection = true +rla_adapter_lr_multiplier = 0.5 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "delivery-eta" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] +start_epoch = 16 +start_fraction = 0.5 +mode = "consensus_raw" + +[cf_fisd.member_groups] +xgb = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] +lgbm = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] +cat = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/1/report.json new file mode 100644 index 000000000..a02af59d6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/1/report.json @@ -0,0 +1,629 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5138602835910225, + "mae": 0.4030267894268036, + "r2": 0.5050784349441528, + "score": -0.5138602835910225 + }, + "val": { + "rmse": 0.5564338969448529, + "mae": 0.43967634439468384, + "r2": 0.4161449074745178, + "score": -0.5564338969448529 + }, + "test": { + "rmse": 0.5491088203688107, + "mae": 0.4416921138763428, + "r2": 0.43814295530319214, + "score": -0.5491088203688107 + } + }, + "time": "0:15:04.310558", + "diversity": { + "train": { + "mean_centered_corr": -0.020146815106272697, + "mean_pairwise_disagreement": NaN, + "member_std": 0.061014287173748016 + }, + "val": { + "mean_centered_corr": -0.020077988505363464, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06329856067895889 + }, + "test": { + "mean_centered_corr": -0.01972520351409912, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06241019815206528 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/2.toml new file mode 100644 index 000000000..d99f2962a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/2.toml @@ -0,0 +1,158 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 2 +head_selection = true +rla_adapter_lr_multiplier = 0.5 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "delivery-eta" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] +start_epoch = 16 +start_fraction = 0.5 +mode = "consensus_raw" + +[cf_fisd.member_groups] +xgb = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] +lgbm = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] +cat = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/2/report.json new file mode 100644 index 000000000..b225106f7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/2/report.json @@ -0,0 +1,629 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1911, + "metrics": { + "train": { + "rmse": 0.5080919523858995, + "mae": 0.3977190852165222, + "r2": 0.5161275267601013, + "score": -0.5080919523858995 + }, + "val": { + "rmse": 0.5562272527951271, + "mae": 0.43906888365745544, + "r2": 0.416578471660614, + "score": -0.5562272527951271 + }, + "test": { + "rmse": 0.5494697322909359, + "mae": 0.44080713391304016, + "r2": 0.43740415573120117, + "score": -0.5494697322909359 + } + }, + "time": "0:16:37.086669", + "diversity": { + "train": { + "mean_centered_corr": -0.020242199301719666, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0667627602815628 + }, + "val": { + "mean_centered_corr": -0.02033313177525997, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06825815886259079 + }, + "test": { + "mean_centered_corr": -0.020218119025230408, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06682927906513214 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..f1e9f1689 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/0/report.json @@ -0,0 +1,641 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5148489632002486, + "mae": 0.40648993849754333, + "r2": 0.5031721591949463, + "score": -0.5148489632002486 + }, + "val": { + "rmse": 0.5553930389270275, + "mae": 0.4419495463371277, + "r2": 0.4183272123336792, + "score": -0.5553930389270275 + }, + "test": { + "rmse": 0.5519483119271499, + "mae": 0.4472074508666992, + "r2": 0.4323171377182007, + "score": -0.5519483119271499 + } + }, + "time": "0:15:04.980270", + "diversity": { + "train": { + "mean_centered_corr": -0.02016017585992813, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06105273962020874 + }, + "val": { + "mean_centered_corr": -0.01993357017636299, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06221357360482216 + }, + "test": { + "mean_centered_corr": -0.020221160724759102, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06099524348974228 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 11, + 10, + 26, + 21, + 15, + 16, + 31, + 19, + 13 + ], + "head_selection_time": "0:00:00.266220" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/1/report.json new file mode 100644 index 000000000..486356000 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/1/report.json @@ -0,0 +1,642 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5146326283410019, + "mae": 0.4055343568325043, + "r2": 0.5035895109176636, + "score": -0.5146326283410019 + }, + "val": { + "rmse": 0.5557605143473513, + "mae": 0.4405376613140106, + "r2": 0.4175572395324707, + "score": -0.5557605143473513 + }, + "test": { + "rmse": 0.5514412571081766, + "mae": 0.4451921284198761, + "r2": 0.43335968255996704, + "score": -0.5514412571081766 + } + }, + "time": "0:15:04.310558", + "diversity": { + "train": { + "mean_centered_corr": -0.020146815106272697, + "mean_pairwise_disagreement": NaN, + "member_std": 0.061014287173748016 + }, + "val": { + "mean_centered_corr": -0.020077988505363464, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06329856067895889 + }, + "test": { + "mean_centered_corr": -0.01972520351409912, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06241019815206528 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 5, + 12, + 30, + 22, + 29, + 18, + 31, + 25, + 4, + 14 + ], + "head_selection_time": "0:00:00.299075" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/2/report.json new file mode 100644 index 000000000..5afeef8d4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/2/report.json @@ -0,0 +1,647 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1911, + "metrics": { + "train": { + "rmse": 0.5091281866287168, + "mae": 0.40107253193855286, + "r2": 0.5141518115997314, + "score": -0.5091281866287168 + }, + "val": { + "rmse": 0.5554844141815782, + "mae": 0.4408845603466034, + "r2": 0.41813576221466064, + "score": -0.5554844141815782 + }, + "test": { + "rmse": 0.5520847398512095, + "mae": 0.44570744037628174, + "r2": 0.4320363998413086, + "score": -0.5520847398512095 + } + }, + "time": "0:16:37.086669", + "diversity": { + "train": { + "mean_centered_corr": -0.020242199301719666, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0667627602815628 + }, + "val": { + "mean_centered_corr": -0.02033313177525997, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06825815886259079 + }, + "test": { + "mean_centered_corr": -0.020218119025230408, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06682927906513214 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 5, + 14, + 23, + 13, + 0, + 24, + 7, + 16, + 29, + 8, + 6, + 14, + 27, + 7, + 18 + ], + "head_selection_time": "0:00:00.472757" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..2b84e1465 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-best-head-evaluation/0/report.json @@ -0,0 +1,229 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2730, + "metrics": { + "train": { + "rmse": 0.5056157353876229, + "mae": 0.3998609483242035, + "r2": 0.5208324193954468, + "score": -0.5056157353876229 + }, + "val": { + "rmse": 0.5596782586864594, + "mae": 0.44516822695732117, + "r2": 0.4093165993690491, + "score": -0.5596782586864594 + }, + "test": { + "rmse": 0.5565908577070521, + "mae": 0.45004433393478394, + "r2": 0.42272716760635376, + "score": -0.5565908577070521 + } + }, + "time": "0:13:12.089007", + "diversity": { + "train": { + "mean_centered_corr": -0.020204994827508926, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07431785017251968 + }, + "val": { + "mean_centered_corr": -0.020286401733756065, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07910650968551636 + }, + "test": { + "mean_centered_corr": -0.02015332505106926, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07623301446437836 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 24 + ], + "head_selection_time": "0:00:00.024366" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-best-head-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-best-head-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-best-head-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-best-head-evaluation/1/report.json new file mode 100644 index 000000000..50d3cc1a2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-best-head-evaluation/1/report.json @@ -0,0 +1,229 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__1.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5157048423679549, + "mae": 0.408806711435318, + "r2": 0.5015189051628113, + "score": -0.5157048423679549 + }, + "val": { + "rmse": 0.5578575885405295, + "mae": 0.44588109850883484, + "r2": 0.41315340995788574, + "score": -0.5578575885405295 + }, + "test": { + "rmse": 0.5568376961917461, + "mae": 0.4522704780101776, + "r2": 0.4222150444984436, + "score": -0.5568376961917461 + } + }, + "time": "0:10:49.864209", + "diversity": { + "train": { + "mean_centered_corr": -0.02029341645538807, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06168676167726517 + }, + "val": { + "mean_centered_corr": -0.020232515409588814, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0631292313337326 + }, + "test": { + "mean_centered_corr": -0.020325176417827606, + "mean_pairwise_disagreement": NaN, + "member_std": 0.061680618673563004 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 22 + ], + "head_selection_time": "0:00:00.024983" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-best-head-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-best-head-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-best-head-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-best-head-evaluation/2/report.json new file mode 100644 index 000000000..af23e2c48 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-best-head-evaluation/2/report.json @@ -0,0 +1,229 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__2.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5148308736438392, + "mae": 0.40706372261047363, + "r2": 0.5032070279121399, + "score": -0.5148308736438392 + }, + "val": { + "rmse": 0.5576450053690213, + "mae": 0.4445996880531311, + "r2": 0.4136006236076355, + "score": -0.5576450053690213 + }, + "test": { + "rmse": 0.5574656479348947, + "mae": 0.45258787274360657, + "r2": 0.42091113328933716, + "score": -0.5574656479348947 + } + }, + "time": "0:10:43.789154", + "diversity": { + "train": { + "mean_centered_corr": -0.0205259807407856, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06119334325194359 + }, + "val": { + "mean_centered_corr": -0.020698808133602142, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06522355228662491 + }, + "test": { + "mean_centered_corr": -0.020667163655161858, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06273415684700012 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 8 + ], + "head_selection_time": "0:00:00.023410" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/0.toml new file mode 100644 index 000000000..474196b1e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/0.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.001 +esam_eps = 1e-12 +esam_adapter_only = false +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 8 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/0/report.json new file mode 100644 index 000000000..f91f1eae8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/0/report.json @@ -0,0 +1,225 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2730, + "metrics": { + "train": { + "rmse": 0.4975037099949126, + "mae": 0.39285650849342346, + "r2": 0.5360844135284424, + "score": -0.4975037099949126 + }, + "val": { + "rmse": 0.5558269510040955, + "mae": 0.44211816787719727, + "r2": 0.4174179434776306, + "score": -0.5558269510040955 + }, + "test": { + "rmse": 0.5527269612276788, + "mae": 0.44693246483802795, + "r2": 0.4307142496109009, + "score": -0.5527269612276788 + } + }, + "time": "0:13:12.089007", + "diversity": { + "train": { + "mean_centered_corr": -0.020204994827508926, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07431785017251968 + }, + "val": { + "mean_centered_corr": -0.020286401733756065, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07910650968551636 + }, + "test": { + "mean_centered_corr": -0.02015332505106926, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07623301446437836 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/1.toml new file mode 100644 index 000000000..d8c270d0c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/1.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 1 +head_selection = true +use_esam = true +esam_rho = 0.001 +esam_eps = 1e-12 +esam_adapter_only = false +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 8 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/1/report.json new file mode 100644 index 000000000..dcf55c4c4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/1/report.json @@ -0,0 +1,225 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__1.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5105330146478386, + "mae": 0.40035760402679443, + "r2": 0.5114669799804688, + "score": -0.5105330146478386 + }, + "val": { + "rmse": 0.5562247345576334, + "mae": 0.439607709646225, + "r2": 0.416583776473999, + "score": -0.5562247345576334 + }, + "test": { + "rmse": 0.550284872190118, + "mae": 0.44156232476234436, + "r2": 0.4357336759567261, + "score": -0.550284872190118 + } + }, + "time": "0:10:49.864209", + "diversity": { + "train": { + "mean_centered_corr": -0.02029341645538807, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06168676167726517 + }, + "val": { + "mean_centered_corr": -0.020232515409588814, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0631292313337326 + }, + "test": { + "mean_centered_corr": -0.020325176417827606, + "mean_pairwise_disagreement": NaN, + "member_std": 0.061680618673563004 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/2.toml new file mode 100644 index 000000000..dedbfb366 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/2.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 2 +head_selection = true +use_esam = true +esam_rho = 0.001 +esam_eps = 1e-12 +esam_adapter_only = false +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 8 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/2/report.json new file mode 100644 index 000000000..fb0de8ba8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/2/report.json @@ -0,0 +1,225 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__2.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5116329266744724, + "mae": 0.4038048982620239, + "r2": 0.5093597173690796, + "score": -0.5116329266744724 + }, + "val": { + "rmse": 0.5557493871736087, + "mae": 0.4417022466659546, + "r2": 0.4175804853439331, + "score": -0.5557493871736087 + }, + "test": { + "rmse": 0.5507763801521746, + "mae": 0.44514498114585876, + "r2": 0.43472522497177124, + "score": -0.5507763801521746 + } + }, + "time": "0:10:43.789154", + "diversity": { + "train": { + "mean_centered_corr": -0.0205259807407856, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06119334325194359 + }, + "val": { + "mean_centered_corr": -0.020698808133602142, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06522355228662491 + }, + "test": { + "mean_centered_corr": -0.020667163655161858, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06273415684700012 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..37c29a8bb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,252 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2730, + "metrics": { + "train": { + "rmse": 0.4981544378477528, + "mae": 0.3936895430088043, + "r2": 0.5348700881004333, + "score": -0.4981544378477528 + }, + "val": { + "rmse": 0.555452544543325, + "mae": 0.44208067655563354, + "r2": 0.4182025194168091, + "score": -0.555452544543325 + }, + "test": { + "rmse": 0.5532771704027001, + "mae": 0.44795727729797363, + "r2": 0.42958033084869385, + "score": -0.5532771704027001 + } + }, + "time": "0:13:12.089007", + "diversity": { + "train": { + "mean_centered_corr": -0.020204994827508926, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07431785017251968 + }, + "val": { + "mean_centered_corr": -0.020286401733756065, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07910650968551636 + }, + "test": { + "mean_centered_corr": -0.02015332505106926, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07623301446437836 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 24, + 31, + 15, + 7, + 13, + 8, + 1, + 27, + 26, + 12, + 10, + 28, + 19, + 2, + 31, + 7, + 13, + 15, + 5, + 8, + 21, + 1, + 4, + 31 + ], + "head_selection_time": "0:00:00.901734" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/1/report.json new file mode 100644 index 000000000..e9b80c6b8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/1/report.json @@ -0,0 +1,237 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__1.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5108096659852551, + "mae": 0.4014206528663635, + "r2": 0.5109373331069946, + "score": -0.5108096659852551 + }, + "val": { + "rmse": 0.5553836483690917, + "mae": 0.440256267786026, + "r2": 0.4183468818664551, + "score": -0.5553836483690917 + }, + "test": { + "rmse": 0.5515298017800222, + "mae": 0.4441634714603424, + "r2": 0.433177649974823, + "score": -0.5515298017800222 + } + }, + "time": "0:10:49.864209", + "diversity": { + "train": { + "mean_centered_corr": -0.02029341645538807, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06168676167726517 + }, + "val": { + "mean_centered_corr": -0.020232515409588814, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0631292313337326 + }, + "test": { + "mean_centered_corr": -0.020325176417827606, + "mean_pairwise_disagreement": NaN, + "member_std": 0.061680618673563004 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 22, + 30, + 29, + 19, + 11, + 18, + 30, + 4, + 16 + ], + "head_selection_time": "0:00:00.270361" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/2/report.json new file mode 100644 index 000000000..29d84861d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/2/report.json @@ -0,0 +1,237 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__2.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5120909698461144, + "mae": 0.4043644666671753, + "r2": 0.5084807872772217, + "score": -0.5120909698461144 + }, + "val": { + "rmse": 0.5554437183690072, + "mae": 0.4415339529514313, + "r2": 0.41822099685668945, + "score": -0.5554437183690072 + }, + "test": { + "rmse": 0.5516331354574737, + "mae": 0.44613465666770935, + "r2": 0.4329652190208435, + "score": -0.5516331354574737 + } + }, + "time": "0:10:43.789154", + "diversity": { + "train": { + "mean_centered_corr": -0.0205259807407856, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06119334325194359 + }, + "val": { + "mean_centered_corr": -0.020698808133602142, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06522355228662491 + }, + "test": { + "mean_centered_corr": -0.020667163655161858, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06273415684700012 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 8, + 31, + 22, + 14, + 12, + 0, + 16, + 27, + 13 + ], + "head_selection_time": "0:00:00.259911" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..0ef052905 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-best-head-evaluation/0/report.json @@ -0,0 +1,212 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" + }, + "mask_stats": { + "keep_rate_actual": 0.9772727489471436, + "min_features_per_member": 210.0, + "mean_features_per_member": 215.0, + "max_features_per_member": 220.0, + "min_members_per_feature": 29.0, + "mean_members_per_feature": 31.272727966308594, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.955511449861677, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.00909090909090909, + "n_core_features": 2.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5081016011029494, + "mae": 0.4003114700317383, + "r2": 0.5161091685295105, + "score": -0.5081016011029494 + }, + "val": { + "rmse": 0.5590051856389011, + "mae": 0.4407220184803009, + "r2": 0.41073644161224365, + "score": -0.5590051856389011 + }, + "test": { + "rmse": 0.5534488118080276, + "mae": 0.4457874596118927, + "r2": 0.4292263984680176, + "score": -0.5534488118080276 + } + }, + "time": "0:07:43.476808", + "diversity": { + "train": { + "mean_centered_corr": -0.02009296789765358, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07790226489305496 + }, + "val": { + "mean_centered_corr": -0.02050519548356533, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07800902426242828 + }, + "test": { + "mean_centered_corr": -0.02043369971215725, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07650945335626602 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 6 + ], + "head_selection_time": "0:00:00.023785" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-best-head-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-best-head-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-best-head-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-best-head-evaluation/1/report.json new file mode 100644 index 000000000..9d0a6ad7c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-best-head-evaluation/1/report.json @@ -0,0 +1,212 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" + }, + "mask_stats": { + "keep_rate_actual": 0.9758522510528564, + "min_features_per_member": 208.0, + "mean_features_per_member": 214.6875, + "max_features_per_member": 219.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.227272033691406, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9528503117194804, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.00909090909090909, + "n_core_features": 2.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5160363608272199, + "mae": 0.40823259949684143, + "r2": 0.5008777976036072, + "score": -0.5160363608272199 + }, + "val": { + "rmse": 0.5576769100347859, + "mae": 0.4429268538951874, + "r2": 0.4135335087776184, + "score": -0.5576769100347859 + }, + "test": { + "rmse": 0.5558933529149844, + "mae": 0.4492851793766022, + "r2": 0.42417311668395996, + "score": -0.5558933529149844 + } + }, + "time": "0:07:03.176224", + "diversity": { + "train": { + "mean_centered_corr": -0.019072668626904488, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07286733388900757 + }, + "val": { + "mean_centered_corr": -0.019180510193109512, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07150229066610336 + }, + "test": { + "mean_centered_corr": -0.01909942366182804, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07076001167297363 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 22 + ], + "head_selection_time": "0:00:00.023446" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-best-head-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-best-head-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-best-head-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-best-head-evaluation/2/report.json new file mode 100644 index 000000000..c6bdbbf0a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-best-head-evaluation/2/report.json @@ -0,0 +1,212 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" + }, + "mask_stats": { + "keep_rate_actual": 0.9745738506317139, + "min_features_per_member": 207.0, + "mean_features_per_member": 214.40625, + "max_features_per_member": 219.0, + "min_members_per_feature": 28.0, + "mean_members_per_feature": 31.186363220214844, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9505048664860526, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.00909090909090909, + "n_core_features": 2.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5149676728055901, + "mae": 0.40653571486473083, + "r2": 0.5029430389404297, + "score": -0.5149676728055901 + }, + "val": { + "rmse": 0.558478390179117, + "mae": 0.44451311230659485, + "r2": 0.4118465781211853, + "score": -0.558478390179117 + }, + "test": { + "rmse": 0.5526173066624255, + "mae": 0.4467785954475403, + "r2": 0.4309401512145996, + "score": -0.5526173066624255 + } + }, + "time": "0:07:03.506041", + "diversity": { + "train": { + "mean_centered_corr": 0.024787528440356255, + "mean_pairwise_disagreement": NaN, + "member_std": 0.11345301568508148 + }, + "val": { + "mean_centered_corr": 0.015488767065107822, + "mean_pairwise_disagreement": NaN, + "member_std": 0.11428248137235641 + }, + "test": { + "mean_centered_corr": 0.020906968042254448, + "mean_pairwise_disagreement": NaN, + "member_std": 0.11407273262739182 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 10 + ], + "head_selection_time": "0:00:00.023247" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/0.toml new file mode 100644 index 000000000..ba5fa4110 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/0.toml @@ -0,0 +1,50 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 +start_epoch = 16 +start_fraction = 0.5 +group_mode = "numerical_only" +categorical_handling = "no_cat_drop" + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/0/report.json new file mode 100644 index 000000000..c3225bc3f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/0/report.json @@ -0,0 +1,208 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" + }, + "mask_stats": { + "keep_rate_actual": 0.9772727489471436, + "min_features_per_member": 210.0, + "mean_features_per_member": 215.0, + "max_features_per_member": 220.0, + "min_members_per_feature": 29.0, + "mean_members_per_feature": 31.272727966308594, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.955511449861677, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.00909090909090909, + "n_core_features": 2.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5047748974218713, + "mae": 0.39609476923942566, + "r2": 0.5224248170852661, + "score": -0.5047748974218713 + }, + "val": { + "rmse": 0.5558048331442785, + "mae": 0.43993815779685974, + "r2": 0.41746431589126587, + "score": -0.5558048331442785 + }, + "test": { + "rmse": 0.5501619198178428, + "mae": 0.44212156534194946, + "r2": 0.435985803604126, + "score": -0.5501619198178428 + } + }, + "time": "0:07:43.476808", + "diversity": { + "train": { + "mean_centered_corr": -0.02009296789765358, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07790226489305496 + }, + "val": { + "mean_centered_corr": -0.02050519548356533, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07800902426242828 + }, + "test": { + "mean_centered_corr": -0.02043369971215725, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07650945335626602 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/1.toml new file mode 100644 index 000000000..27698723e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/1.toml @@ -0,0 +1,50 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 1 +head_selection = true + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 1 +start_epoch = 16 +start_fraction = 0.5 +group_mode = "numerical_only" +categorical_handling = "no_cat_drop" + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/1/report.json new file mode 100644 index 000000000..a4b87cbab --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/1/report.json @@ -0,0 +1,208 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" + }, + "mask_stats": { + "keep_rate_actual": 0.9758522510528564, + "min_features_per_member": 208.0, + "mean_features_per_member": 214.6875, + "max_features_per_member": 219.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.227272033691406, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9528503117194804, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.00909090909090909, + "n_core_features": 2.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5110728431654283, + "mae": 0.4014737606048584, + "r2": 0.5104333162307739, + "score": -0.5110728431654283 + }, + "val": { + "rmse": 0.5559520276955289, + "mae": 0.43929943442344666, + "r2": 0.41715574264526367, + "score": -0.5559520276955289 + }, + "test": { + "rmse": 0.5503973468399834, + "mae": 0.44218185544013977, + "r2": 0.4355030059814453, + "score": -0.5503973468399834 + } + }, + "time": "0:07:03.176224", + "diversity": { + "train": { + "mean_centered_corr": -0.019072668626904488, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07286733388900757 + }, + "val": { + "mean_centered_corr": -0.019180510193109512, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07150229066610336 + }, + "test": { + "mean_centered_corr": -0.01909942366182804, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07076001167297363 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/2.toml new file mode 100644 index 000000000..f4f39c2bd --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/2.toml @@ -0,0 +1,50 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 2 +head_selection = true + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 2 +start_epoch = 16 +start_fraction = 0.5 +group_mode = "numerical_only" +categorical_handling = "no_cat_drop" + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/2/report.json new file mode 100644 index 000000000..fff0b3972 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/2/report.json @@ -0,0 +1,208 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" + }, + "mask_stats": { + "keep_rate_actual": 0.9745738506317139, + "min_features_per_member": 207.0, + "mean_features_per_member": 214.40625, + "max_features_per_member": 219.0, + "min_members_per_feature": 28.0, + "mean_members_per_feature": 31.186363220214844, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9505048664860526, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.00909090909090909, + "n_core_features": 2.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5118594085498649, + "mae": 0.4030582904815674, + "r2": 0.508925199508667, + "score": -0.5118594085498649 + }, + "val": { + "rmse": 0.5562502378827562, + "mae": 0.4414753019809723, + "r2": 0.4165303111076355, + "score": -0.5562502378827562 + }, + "test": { + "rmse": 0.5500394546922427, + "mae": 0.44385185837745667, + "r2": 0.4362368583679199, + "score": -0.5500394546922427 + } + }, + "time": "0:07:03.506041", + "diversity": { + "train": { + "mean_centered_corr": 0.024787528440356255, + "mean_pairwise_disagreement": NaN, + "member_std": 0.11345301568508148 + }, + "val": { + "mean_centered_corr": 0.015488767065107822, + "mean_pairwise_disagreement": NaN, + "member_std": 0.11428248137235641 + }, + "test": { + "mean_centered_corr": 0.020906968042254448, + "mean_pairwise_disagreement": NaN, + "member_std": 0.11407273262739182 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..798a74fc5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,224 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" + }, + "mask_stats": { + "keep_rate_actual": 0.9772727489471436, + "min_features_per_member": 210.0, + "mean_features_per_member": 215.0, + "max_features_per_member": 220.0, + "min_members_per_feature": 29.0, + "mean_members_per_feature": 31.272727966308594, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.955511449861677, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.00909090909090909, + "n_core_features": 2.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5051252716316714, + "mae": 0.39751043915748596, + "r2": 0.5217615962028503, + "score": -0.5051252716316714 + }, + "val": { + "rmse": 0.5553099940581016, + "mae": 0.44019240140914917, + "r2": 0.4185011386871338, + "score": -0.5553099940581016 + }, + "test": { + "rmse": 0.5510631408064661, + "mae": 0.44413861632347107, + "r2": 0.43413645029067993, + "score": -0.5510631408064661 + } + }, + "time": "0:07:43.476808", + "diversity": { + "train": { + "mean_centered_corr": -0.02009296789765358, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07790226489305496 + }, + "val": { + "mean_centered_corr": -0.02050519548356533, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07800902426242828 + }, + "test": { + "mean_centered_corr": -0.02043369971215725, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07650945335626602 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 6, + 13, + 26, + 10, + 14, + 8, + 7, + 27, + 17, + 26, + 13, + 25, + 19 + ], + "head_selection_time": "0:00:00.388499" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/1/report.json new file mode 100644 index 000000000..c2a3902a9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/1/report.json @@ -0,0 +1,220 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" + }, + "mask_stats": { + "keep_rate_actual": 0.9758522510528564, + "min_features_per_member": 208.0, + "mean_features_per_member": 214.6875, + "max_features_per_member": 219.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.227272033691406, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9528503117194804, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.00909090909090909, + "n_core_features": 2.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5118196984810967, + "mae": 0.404056578874588, + "r2": 0.5090013742446899, + "score": -0.5118196984810967 + }, + "val": { + "rmse": 0.55527543085353, + "mae": 0.4401775598526001, + "r2": 0.4185735583305359, + "score": -0.55527543085353 + }, + "test": { + "rmse": 0.5524079939671243, + "mae": 0.44586870074272156, + "r2": 0.43137115240097046, + "score": -0.5524079939671243 + } + }, + "time": "0:07:03.176224", + "diversity": { + "train": { + "mean_centered_corr": -0.019072668626904488, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07286733388900757 + }, + "val": { + "mean_centered_corr": -0.019180510193109512, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07150229066610336 + }, + "test": { + "mean_centered_corr": -0.01909942366182804, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07076001167297363 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 22, + 30, + 4, + 21, + 18, + 19, + 23, + 30, + 1 + ], + "head_selection_time": "0:00:00.258532" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/2/report.json new file mode 100644 index 000000000..5eb17c255 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/2/report.json @@ -0,0 +1,231 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" + }, + "mask_stats": { + "keep_rate_actual": 0.9745738506317139, + "min_features_per_member": 207.0, + "mean_features_per_member": 214.40625, + "max_features_per_member": 219.0, + "min_members_per_feature": 28.0, + "mean_members_per_feature": 31.186363220214844, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9505048664860526, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.00909090909090909, + "n_core_features": 2.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5126941208598502, + "mae": 0.404828816652298, + "r2": 0.5073222517967224, + "score": -0.5126941208598502 + }, + "val": { + "rmse": 0.5553606274153348, + "mae": 0.4412638247013092, + "r2": 0.4183950424194336, + "score": -0.5553606274153348 + }, + "test": { + "rmse": 0.5524157087402433, + "mae": 0.4469607174396515, + "r2": 0.43135523796081543, + "score": -0.5524157087402433 + } + }, + "time": "0:07:03.506041", + "diversity": { + "train": { + "mean_centered_corr": 0.024787528440356255, + "mean_pairwise_disagreement": NaN, + "member_std": 0.11345301568508148 + }, + "val": { + "mean_centered_corr": 0.015488767065107822, + "mean_pairwise_disagreement": NaN, + "member_std": 0.11428248137235641 + }, + "test": { + "mean_centered_corr": 0.020906968042254448, + "mean_pairwise_disagreement": NaN, + "member_std": 0.11407273262739182 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 10, + 14, + 4, + 6, + 27, + 13, + 8, + 31, + 16, + 12, + 14, + 25, + 28, + 4, + 1, + 27, + 14, + 31, + 4, + 6 + ], + "head_selection_time": "0:00:00.694336" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..f6fc58a42 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5175918577279733, + "mae": 0.40775081515312195, + "r2": 0.4978642463684082, + "score": -0.5175918577279733 + }, + "val": { + "rmse": 0.5572652691682626, + "mae": 0.44076380133628845, + "r2": 0.4143989086151123, + "score": -0.5572652691682626 + }, + "test": { + "rmse": 0.5524207799310179, + "mae": 0.4452172815799713, + "r2": 0.4313448667526245, + "score": -0.5524207799310179 + } + }, + "time": "0:14:43.732239", + "diversity": { + "train": { + "mean_centered_corr": -0.02016017585992813, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06105273962020874 + }, + "val": { + "mean_centered_corr": -0.01993357017636299, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06221357360482216 + }, + "test": { + "mean_centered_corr": -0.020221160724759102, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06099524348974228 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 11 + ], + "head_selection_time": "0:00:00.024300" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/1/report.json new file mode 100644 index 000000000..b01d46298 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/1/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5171067304438404, + "mae": 0.40677744150161743, + "r2": 0.49880504608154297, + "score": -0.5171067304438404 + }, + "val": { + "rmse": 0.5577909128836495, + "mae": 0.44260263442993164, + "r2": 0.41329365968704224, + "score": -0.5577909128836495 + }, + "test": { + "rmse": 0.5532065757037241, + "mae": 0.44593191146850586, + "r2": 0.4297259449958801, + "score": -0.5532065757037241 + } + }, + "time": "0:14:46.412006", + "diversity": { + "train": { + "mean_centered_corr": -0.020146815106272697, + "mean_pairwise_disagreement": NaN, + "member_std": 0.061014287173748016 + }, + "val": { + "mean_centered_corr": -0.020077988505363464, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06329856067895889 + }, + "test": { + "mean_centered_corr": -0.01972520351409912, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06241019815206528 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 5 + ], + "head_selection_time": "0:00:00.023596" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/2/report.json new file mode 100644 index 000000000..7c2f9848a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/2/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1911, + "metrics": { + "train": { + "rmse": 0.5137978750972679, + "mae": 0.4069885015487671, + "r2": 0.50519859790802, + "score": -0.5137978750972679 + }, + "val": { + "rmse": 0.5585470113340857, + "mae": 0.44582831859588623, + "r2": 0.4117019772529602, + "score": -0.5585470113340857 + }, + "test": { + "rmse": 0.5573737419749005, + "mae": 0.4522700607776642, + "r2": 0.4211021065711975, + "score": -0.5573737419749005 + } + }, + "time": "0:16:06.202820", + "diversity": { + "train": { + "mean_centered_corr": -0.020242199301719666, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0667627602815628 + }, + "val": { + "mean_centered_corr": -0.02033313177525997, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06825815886259079 + }, + "test": { + "mean_centered_corr": -0.020218119025230408, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06682927906513214 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 5 + ], + "head_selection_time": "0:00:00.024826" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/0.toml new file mode 100644 index 000000000..f15a53b07 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.5 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/0/report.json new file mode 100644 index 000000000..9c3fd8d6d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5150130136938161, + "mae": 0.40628108382225037, + "r2": 0.5028554797172546, + "score": -0.5150130136938161 + }, + "val": { + "rmse": 0.5559356776481553, + "mae": 0.441616415977478, + "r2": 0.4171900153160095, + "score": -0.5559356776481553 + }, + "test": { + "rmse": 0.55135486033622, + "mae": 0.445721298456192, + "r2": 0.43353718519210815, + "score": -0.55135486033622 + } + }, + "time": "0:14:43.732239", + "diversity": { + "train": { + "mean_centered_corr": -0.02016017585992813, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06105273962020874 + }, + "val": { + "mean_centered_corr": -0.01993357017636299, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06221357360482216 + }, + "test": { + "mean_centered_corr": -0.020221160724759102, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06099524348974228 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/1.toml new file mode 100644 index 000000000..19367e222 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/1.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 1 +head_selection = true +rla_adapter_lr_multiplier = 0.5 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/1/report.json new file mode 100644 index 000000000..941521617 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/1/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5138602835910225, + "mae": 0.4030267894268036, + "r2": 0.5050784349441528, + "score": -0.5138602835910225 + }, + "val": { + "rmse": 0.5564338969448529, + "mae": 0.43967634439468384, + "r2": 0.4161449074745178, + "score": -0.5564338969448529 + }, + "test": { + "rmse": 0.5491088203688107, + "mae": 0.4416921138763428, + "r2": 0.43814295530319214, + "score": -0.5491088203688107 + } + }, + "time": "0:14:46.412006", + "diversity": { + "train": { + "mean_centered_corr": -0.020146815106272697, + "mean_pairwise_disagreement": NaN, + "member_std": 0.061014287173748016 + }, + "val": { + "mean_centered_corr": -0.020077988505363464, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06329856067895889 + }, + "test": { + "mean_centered_corr": -0.01972520351409912, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06241019815206528 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/2.toml new file mode 100644 index 000000000..94b60b1ef --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/2.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 2 +head_selection = true +rla_adapter_lr_multiplier = 0.5 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/2/report.json new file mode 100644 index 000000000..33f34dd36 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/2/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1911, + "metrics": { + "train": { + "rmse": 0.5080919523858995, + "mae": 0.3977190852165222, + "r2": 0.5161275267601013, + "score": -0.5080919523858995 + }, + "val": { + "rmse": 0.5562272527951271, + "mae": 0.43906888365745544, + "r2": 0.416578471660614, + "score": -0.5562272527951271 + }, + "test": { + "rmse": 0.5494697322909359, + "mae": 0.44080713391304016, + "r2": 0.43740415573120117, + "score": -0.5494697322909359 + } + }, + "time": "0:16:06.202820", + "diversity": { + "train": { + "mean_centered_corr": -0.020242199301719666, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0667627602815628 + }, + "val": { + "mean_centered_corr": -0.02033313177525997, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06825815886259079 + }, + "test": { + "mean_centered_corr": -0.020218119025230408, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06682927906513214 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..043c34d52 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,187 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5148489632002486, + "mae": 0.40648993849754333, + "r2": 0.5031721591949463, + "score": -0.5148489632002486 + }, + "val": { + "rmse": 0.5553930389270275, + "mae": 0.4419495463371277, + "r2": 0.4183272123336792, + "score": -0.5553930389270275 + }, + "test": { + "rmse": 0.5519483119271499, + "mae": 0.4472074508666992, + "r2": 0.4323171377182007, + "score": -0.5519483119271499 + } + }, + "time": "0:14:43.732239", + "diversity": { + "train": { + "mean_centered_corr": -0.02016017585992813, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06105273962020874 + }, + "val": { + "mean_centered_corr": -0.01993357017636299, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06221357360482216 + }, + "test": { + "mean_centered_corr": -0.020221160724759102, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06099524348974228 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 11, + 10, + 26, + 21, + 15, + 16, + 31, + 19, + 13 + ], + "head_selection_time": "0:00:00.265073" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/1/report.json new file mode 100644 index 000000000..437652c9c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/1/report.json @@ -0,0 +1,188 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5146326283410019, + "mae": 0.4055343568325043, + "r2": 0.5035895109176636, + "score": -0.5146326283410019 + }, + "val": { + "rmse": 0.5557605143473513, + "mae": 0.4405376613140106, + "r2": 0.4175572395324707, + "score": -0.5557605143473513 + }, + "test": { + "rmse": 0.5514412571081766, + "mae": 0.4451921284198761, + "r2": 0.43335968255996704, + "score": -0.5514412571081766 + } + }, + "time": "0:14:46.412006", + "diversity": { + "train": { + "mean_centered_corr": -0.020146815106272697, + "mean_pairwise_disagreement": NaN, + "member_std": 0.061014287173748016 + }, + "val": { + "mean_centered_corr": -0.020077988505363464, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06329856067895889 + }, + "test": { + "mean_centered_corr": -0.01972520351409912, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06241019815206528 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 5, + 12, + 30, + 22, + 29, + 18, + 31, + 25, + 4, + 14 + ], + "head_selection_time": "0:00:00.290840" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/2/report.json new file mode 100644 index 000000000..c3c6e1a67 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/2/report.json @@ -0,0 +1,193 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1911, + "metrics": { + "train": { + "rmse": 0.5091281866287168, + "mae": 0.40107253193855286, + "r2": 0.5141518115997314, + "score": -0.5091281866287168 + }, + "val": { + "rmse": 0.5554844141815782, + "mae": 0.4408845603466034, + "r2": 0.41813576221466064, + "score": -0.5554844141815782 + }, + "test": { + "rmse": 0.5520847398512095, + "mae": 0.44570744037628174, + "r2": 0.4320363998413086, + "score": -0.5520847398512095 + } + }, + "time": "0:16:06.202820", + "diversity": { + "train": { + "mean_centered_corr": -0.020242199301719666, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0667627602815628 + }, + "val": { + "mean_centered_corr": -0.02033313177525997, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06825815886259079 + }, + "test": { + "mean_centered_corr": -0.020218119025230408, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06682927906513214 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 5, + 14, + 23, + 13, + 0, + 24, + 7, + 16, + 29, + 8, + 6, + 14, + 27, + 7, + 18 + ], + "head_selection_time": "0:00:00.488015" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/0/report.json new file mode 100644 index 000000000..0da12dd14 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/0/report.json @@ -0,0 +1,219 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7835229427379597, + "recall": 0.9817888071846897, + "f1-score": 0.871522047464133, + "support": 84179.0 + }, + "1": { + "precision": 0.602952602952603, + "recall": 0.09252046737143312, + "f1-score": 0.1604244909209937, + "support": 25162.0 + }, + "accuracy": 0.7771467244674916, + "macro avg": { + "precision": 0.6932377728452814, + "recall": 0.5371546372780615, + "f1-score": 0.5159732691925634, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7419693545168977, + "recall": 0.7771467244674916, + "f1-score": 0.707881357167369, + "support": 109341.0 + }, + "cross-entropy": 0.49822654924283316, + "roc-auc": 0.6867900811541505, + "score": 0.6867900811541505 + }, + "val": { + "0": { + "precision": 0.6877843221759325, + "recall": 0.9255703054517335, + "f1-score": 0.7891541441169199, + "support": 15518.0 + }, + "1": { + "precision": 0.6580817051509769, + "recall": 0.2542605512981814, + "f1-score": 0.36680141902483293, + "support": 8743.0 + }, + "accuracy": 0.6836486542187049, + "macro avg": { + "precision": 0.6729330136634547, + "recall": 0.5899154283749575, + "f1-score": 0.5779777815708764, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6770803124216278, + "recall": 0.6836486542187049, + "f1-score": 0.6369497883409784, + "support": 24261.0 + }, + "cross-entropy": 0.6337455659020594, + "roc-auc": 0.6404461738890127, + "score": 0.6404461738890127 + }, + "test": { + "0": { + "precision": 0.6663787430075683, + "recall": 0.9573927431745657, + "f1-score": 0.785807828491051, + "support": 16922.0 + }, + "1": { + "precision": 0.6635557629491368, + "recall": 0.14916605475715933, + "f1-score": 0.24357656731757452, + "support": 9533.0 + }, + "accuracy": 0.6661500661500661, + "macro avg": { + "precision": 0.6649672529783526, + "recall": 0.5532793989658625, + "f1-score": 0.5146921979043128, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6653614884660062, + "recall": 0.6661500661500661, + "f1-score": 0.5904160079366473, + "support": 26455.0 + }, + "cross-entropy": 0.6685406788276543, + "roc-auc": 0.6034781946000056, + "score": 0.6034781946000056 + } + }, + "time": "0:01:00.788883", + "diversity": { + "train": { + "mean_centered_corr": -0.013087093830108643, + "mean_pairwise_disagreement": 0.025822423398123243, + "member_std": 0.02709772437810898 + }, + "val": { + "mean_centered_corr": -0.004288136959075928, + "mean_pairwise_disagreement": 0.08692423855623854, + "member_std": 0.060995232313871384 + }, + "test": { + "mean_centered_corr": 0.0036114430986344814, + "mean_pairwise_disagreement": 0.05469726437468373, + "member_std": 0.05942297354340553 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 14 + ], + "head_selection_time": "0:00:04.762789" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/1/report.json new file mode 100644 index 000000000..baebe29e5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/1/report.json @@ -0,0 +1,219 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 428, + "metrics": { + "train": { + "0": { + "precision": 0.7921504831670823, + "recall": 0.9660247805272099, + "f1-score": 0.8704900045494688, + "support": 84179.0 + }, + "1": { + "precision": 0.5721765145848915, + "recall": 0.1520149431682696, + "f1-score": 0.2402110088862373, + "support": 25162.0 + }, + "accuracy": 0.7787014934928344, + "macro avg": { + "precision": 0.6821634988759868, + "recall": 0.5590198618477398, + "f1-score": 0.555350506717853, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7415291700506383, + "recall": 0.7787014934928344, + "f1-score": 0.7254476134164242, + "support": 109341.0 + }, + "cross-entropy": 0.4932425445384841, + "roc-auc": 0.6956628076755741, + "score": 0.6956628076755741 + }, + "val": { + "0": { + "precision": 0.6824526887764661, + "recall": 0.9388452120118572, + "f1-score": 0.7903759561655727, + "support": 15518.0 + }, + "1": { + "precision": 0.6742190181943014, + "recall": 0.22463685233901406, + "f1-score": 0.3369938229238161, + "support": 8743.0 + }, + "accuracy": 0.6814640781501174, + "macro avg": { + "precision": 0.6783358534853838, + "recall": 0.5817410321754356, + "f1-score": 0.5636848895446944, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6794854993819701, + "recall": 0.6814640781501174, + "f1-score": 0.6269894514488389, + "support": 24261.0 + }, + "cross-entropy": 0.6629784158321528, + "roc-auc": 0.6313621036574809, + "score": 0.6313621036574809 + }, + "test": { + "0": { + "precision": 0.6617878311358413, + "recall": 0.9699208131426545, + "f1-score": 0.7867603000742995, + "support": 16922.0 + }, + "1": { + "precision": 0.6922611850060459, + "recall": 0.12010909472359173, + "f1-score": 0.2047018861178153, + "support": 9533.0 + }, + "accuracy": 0.6636930636930637, + "macro avg": { + "precision": 0.6770245080709436, + "recall": 0.5450149539331232, + "f1-score": 0.49573109309605745, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6727688359532542, + "recall": 0.6636930636930637, + "f1-score": 0.5770168542135109, + "support": 26455.0 + }, + "cross-entropy": 0.7192501404441878, + "roc-auc": 0.5963646574673217, + "score": 0.5963646574673217 + } + }, + "time": "0:01:04.856616", + "diversity": { + "train": { + "mean_centered_corr": -0.009633250534534454, + "mean_pairwise_disagreement": 0.02490320677749485, + "member_std": 0.024462362751364708 + }, + "val": { + "mean_centered_corr": 0.023074615746736526, + "mean_pairwise_disagreement": 0.06982341565049974, + "member_std": 0.0468476265668869 + }, + "test": { + "mean_centered_corr": 0.04183991998434067, + "mean_pairwise_disagreement": 0.04013975344620506, + "member_std": 0.043004557490348816 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 15 + ], + "head_selection_time": "0:00:04.849572" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/2/report.json new file mode 100644 index 000000000..0f03817b9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/2/report.json @@ -0,0 +1,219 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7892154495494365, + "recall": 0.9707052827902446, + "f1-score": 0.87060239936926, + "support": 84179.0 + }, + "1": { + "precision": 0.5751206064782909, + "recall": 0.13266036086161673, + "f1-score": 0.215591293676936, + "support": 25162.0 + }, + "accuracy": 0.7778509433789704, + "macro avg": { + "precision": 0.6821680280138637, + "recall": 0.5516828218259306, + "f1-score": 0.543096846523098, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7399470649420506, + "recall": 0.7778509433789704, + "f1-score": 0.7198685534978095, + "support": 109341.0 + }, + "cross-entropy": 0.49882900198018437, + "roc-auc": 0.6810205996481966, + "score": 0.6810205996481966 + }, + "val": { + "0": { + "precision": 0.6962851665716803, + "recall": 0.8660265498131202, + "f1-score": 0.7719348631494299, + "support": 15518.0 + }, + "1": { + "precision": 0.5808467741935484, + "recall": 0.32952075946471465, + "f1-score": 0.4204918630956725, + "support": 8743.0 + }, + "accuracy": 0.6726845554593792, + "macro avg": { + "precision": 0.6385659703826143, + "recall": 0.5977736546389174, + "f1-score": 0.5962133631225512, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6546843312985255, + "recall": 0.6726845554593792, + "f1-score": 0.6452844303779035, + "support": 24261.0 + }, + "cross-entropy": 0.6307411915692648, + "roc-auc": 0.6342271136151092, + "score": 0.6342271136151092 + }, + "test": { + "0": { + "precision": 0.670945256715403, + "recall": 0.932868455265335, + "f1-score": 0.7805191594561187, + "support": 16922.0 + }, + "1": { + "precision": 0.6118893064571234, + "recall": 0.18787370187768804, + "f1-score": 0.28747993579454256, + "support": 9533.0 + }, + "accuracy": 0.6644112644112644, + "macro avg": { + "precision": 0.6414172815862631, + "recall": 0.5603710785715115, + "f1-score": 0.5339995476253306, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6496645773046232, + "recall": 0.6644112644112644, + "f1-score": 0.6028535794460712, + "support": 26455.0 + }, + "cross-entropy": 0.6521463035772068, + "roc-auc": 0.5992387053088735, + "score": 0.5992387053088735 + } + }, + "time": "0:01:00.829886", + "diversity": { + "train": { + "mean_centered_corr": -0.0152942705899477, + "mean_pairwise_disagreement": 0.023205388676030093, + "member_std": 0.024158649146556854 + }, + "val": { + "mean_centered_corr": -0.00040229607839137316, + "mean_pairwise_disagreement": 0.06998355252223468, + "member_std": 0.05039098858833313 + }, + "test": { + "mean_centered_corr": 0.0061957864090800285, + "mean_pairwise_disagreement": 0.04117994037348876, + "member_std": 0.047571223229169846 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31 + ], + "head_selection_time": "0:00:04.818158" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-evaluation/0.toml new file mode 100644 index 000000000..bf2cd1131 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-evaluation/0.toml @@ -0,0 +1,33 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-evaluation/0/report.json new file mode 100644 index 000000000..9b9138da9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-evaluation/0/report.json @@ -0,0 +1,215 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7862557522292872, + "recall": 0.9783200085532021, + "f1-score": 0.8718353174078054, + "support": 84179.0 + }, + "1": { + "precision": 0.6031746031746031, + "recall": 0.11024560845719736, + "f1-score": 0.18641846712140048, + "support": 25162.0 + }, + "accuracy": 0.7785551622904492, + "macro avg": { + "precision": 0.6947151777019451, + "recall": 0.5442828085051997, + "f1-score": 0.529126892264603, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7441243662668947, + "recall": 0.7785551622904492, + "f1-score": 0.7141043767093801, + "support": 109341.0 + }, + "cross-entropy": 0.4922807370219411, + "roc-auc": 0.6989988975549913, + "score": 0.6989988975549913 + }, + "val": { + "0": { + "precision": 0.6750500272876114, + "recall": 0.9565021265627014, + "f1-score": 0.7915000266624007, + "support": 15518.0 + }, + "1": { + "precision": 0.7030356357237132, + "recall": 0.1827747912615807, + "f1-score": 0.29012345679012347, + "support": 8743.0 + }, + "accuracy": 0.6776719838423808, + "macro avg": { + "precision": 0.6890428315056623, + "recall": 0.569638458912141, + "f1-score": 0.5408117417262621, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6851352741676592, + "recall": 0.6776719838423808, + "f1-score": 0.6108176413364323, + "support": 24261.0 + }, + "cross-entropy": 0.6872792555776136, + "roc-auc": 0.6227054517511603, + "score": 0.6227054517511603 + }, + "test": { + "0": { + "precision": 0.656363924552227, + "recall": 0.9788441082614349, + "f1-score": 0.7858057782627259, + "support": 16922.0 + }, + "1": { + "precision": 0.7063166529942576, + "recall": 0.09031784328123361, + "f1-score": 0.16015625, + "support": 9533.0 + }, + "accuracy": 0.6586656586656586, + "macro avg": { + "precision": 0.6813402887732423, + "recall": 0.5345809757713342, + "f1-score": 0.47298101413136295, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6743642783695727, + "recall": 0.6586656586656586, + "f1-score": 0.5603543719906198, + "support": 26455.0 + }, + "cross-entropy": 0.756433532699498, + "roc-auc": 0.591151981311678, + "score": 0.591151981311678 + } + }, + "time": "0:01:00.788883", + "diversity": { + "train": { + "mean_centered_corr": -0.013087093830108643, + "mean_pairwise_disagreement": 0.025822423398123243, + "member_std": 0.02709772437810898 + }, + "val": { + "mean_centered_corr": -0.004288136959075928, + "mean_pairwise_disagreement": 0.08692423855623854, + "member_std": 0.060995232313871384 + }, + "test": { + "mean_centered_corr": 0.0036114430986344814, + "mean_pairwise_disagreement": 0.05469726437468373, + "member_std": 0.05942297354340553 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-evaluation/1.toml new file mode 100644 index 000000000..48a2a15be --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-evaluation/1.toml @@ -0,0 +1,33 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 1 +head_selection = true + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-evaluation/1/report.json new file mode 100644 index 000000000..e74ec9d76 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-evaluation/1/report.json @@ -0,0 +1,215 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 428, + "metrics": { + "train": { + "0": { + "precision": 0.7880169294701383, + "recall": 0.9754095439480155, + "f1-score": 0.8717564870259481, + "support": 84179.0 + }, + "1": { + "precision": 0.5975894245723172, + "recall": 0.12216834909784596, + "f1-score": 0.202864119316307, + "support": 25162.0 + }, + "accuracy": 0.7790581757986482, + "macro avg": { + "precision": 0.6928031770212277, + "recall": 0.5487889465229308, + "f1-score": 0.5373103031711275, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7441949699285303, + "recall": 0.7790581757986482, + "f1-score": 0.7178282281266332, + "support": 109341.0 + }, + "cross-entropy": 0.490351542028749, + "roc-auc": 0.7032208565488707, + "score": 0.7032208565488707 + }, + "val": { + "0": { + "precision": 0.6735332819093425, + "recall": 0.9565665678566826, + "f1-score": 0.7904784727214633, + "support": 15518.0 + }, + "1": { + "precision": 0.6966696669666966, + "recall": 0.17705593045865264, + "f1-score": 0.2823529411764706, + "support": 8743.0 + }, + "accuracy": 0.6756522814393471, + "macro avg": { + "precision": 0.6851014744380195, + "recall": 0.5668112491576676, + "f1-score": 0.536415706948967, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6818710014821732, + "recall": 0.6756522814393471, + "f1-score": 0.6073639464324451, + "support": 24261.0 + }, + "cross-entropy": 0.713781615665886, + "roc-auc": 0.6203331748306973, + "score": 0.6203331748306973 + }, + "test": { + "0": { + "precision": 0.6556079021338929, + "recall": 0.9786077295827916, + "f1-score": 0.7851876437258482, + "support": 16922.0 + }, + "1": { + "precision": 0.697324414715719, + "recall": 0.0874855764187559, + "f1-score": 0.1554664926833815, + "support": 9533.0 + }, + "accuracy": 0.6574938574938575, + "macro avg": { + "precision": 0.676466158424806, + "recall": 0.5330466530007738, + "f1-score": 0.47032706820461484, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6706403540122731, + "recall": 0.6574938574938575, + "f1-score": 0.5582690373040816, + "support": 26455.0 + }, + "cross-entropy": 0.7978280002692709, + "roc-auc": 0.5897210540664094, + "score": 0.5897210540664094 + } + }, + "time": "0:01:04.856616", + "diversity": { + "train": { + "mean_centered_corr": -0.009633250534534454, + "mean_pairwise_disagreement": 0.02490320677749485, + "member_std": 0.024462362751364708 + }, + "val": { + "mean_centered_corr": 0.023074615746736526, + "mean_pairwise_disagreement": 0.06982341565049974, + "member_std": 0.0468476265668869 + }, + "test": { + "mean_centered_corr": 0.04183991998434067, + "mean_pairwise_disagreement": 0.04013975344620506, + "member_std": 0.043004557490348816 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-evaluation/2.toml new file mode 100644 index 000000000..ab67f52a6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-evaluation/2.toml @@ -0,0 +1,33 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 2 +head_selection = true + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-evaluation/2/report.json new file mode 100644 index 000000000..0f9aca3a7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-evaluation/2/report.json @@ -0,0 +1,215 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.786010743352193, + "recall": 0.978640753632141, + "f1-score": 0.8718119668973692, + "support": 84179.0 + }, + "1": { + "precision": 0.6032656663724625, + "recall": 0.10865590970511088, + "f1-score": 0.18414494510675558, + "support": 25162.0 + }, + "accuracy": 0.7784362681885112, + "macro avg": { + "precision": 0.6946382048623277, + "recall": 0.5436483316686259, + "f1-score": 0.5279784560020624, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7439566956759877, + "recall": 0.7784362681885112, + "f1-score": 0.7135632074906011, + "support": 109341.0 + }, + "cross-entropy": 0.4919100581485962, + "roc-auc": 0.6999783629949486, + "score": 0.6999783629949486 + }, + "val": { + "0": { + "precision": 0.676180922852453, + "recall": 0.955664389740946, + "f1-score": 0.7919893190921228, + "support": 15518.0 + }, + "1": { + "precision": 0.7045942464577072, + "recall": 0.18769301155209883, + "f1-score": 0.2964234104046243, + "support": 8743.0 + }, + "accuracy": 0.678908536334034, + "macro avg": { + "precision": 0.6903875846550801, + "recall": 0.5716787006465225, + "f1-score": 0.5442063647483735, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6864203065662627, + "recall": 0.678908536334034, + "f1-score": 0.6134009369291947, + "support": 24261.0 + }, + "cross-entropy": 0.7026160031895037, + "roc-auc": 0.6212503263524413, + "score": 0.6212503263524413 + }, + "test": { + "0": { + "precision": 0.6575974542561654, + "recall": 0.9769530788322893, + "f1-score": 0.7860776948314393, + "support": 16922.0 + }, + "1": { + "precision": 0.7034220532319392, + "recall": 0.09703136473303263, + "f1-score": 0.17053834808259588, + "support": 9533.0 + }, + "accuracy": 0.6598752598752599, + "macro avg": { + "precision": 0.6805097537440523, + "recall": 0.5369922217826609, + "f1-score": 0.47830802145701756, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6741102458659197, + "recall": 0.6598752598752599, + "f1-score": 0.5642694698245702, + "support": 26455.0 + }, + "cross-entropy": 0.780454852534039, + "roc-auc": 0.5893011087345269, + "score": 0.5893011087345269 + } + }, + "time": "0:01:00.829886", + "diversity": { + "train": { + "mean_centered_corr": -0.0152942705899477, + "mean_pairwise_disagreement": 0.023205388676030093, + "member_std": 0.024158649146556854 + }, + "val": { + "mean_centered_corr": -0.00040229607839137316, + "mean_pairwise_disagreement": 0.06998355252223468, + "member_std": 0.05039098858833313 + }, + "test": { + "mean_centered_corr": 0.0061957864090800285, + "mean_pairwise_disagreement": 0.04117994037348876, + "member_std": 0.047571223229169846 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..5f26c2628 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/0/report.json @@ -0,0 +1,227 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7829606930730526, + "recall": 0.9834281709214887, + "f1-score": 0.8718189046448073, + "support": 84179.0 + }, + "1": { + "precision": 0.6134663341645885, + "recall": 0.08798982592798664, + "f1-score": 0.15390497375829829, + "support": 25162.0 + }, + "accuracy": 0.7773662212710694, + "macro avg": { + "precision": 0.6982135136188206, + "recall": 0.5357089984247376, + "f1-score": 0.5128619392015528, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7439559550621073, + "recall": 0.7773662212710694, + "f1-score": 0.7066096022882681, + "support": 109341.0 + }, + "cross-entropy": 0.4977984361309486, + "roc-auc": 0.6854164410903827, + "score": 0.6854164410903827 + }, + "val": { + "0": { + "precision": 0.6785484611851171, + "recall": 0.9519267946900374, + "f1-score": 0.7923192447972538, + "support": 15518.0 + }, + "1": { + "precision": 0.7005218787635488, + "recall": 0.19958824202218917, + "f1-score": 0.31066405554566495, + "support": 8743.0 + }, + "accuracy": 0.6808045834879024, + "macro avg": { + "precision": 0.689535169974333, + "recall": 0.5757575183561132, + "f1-score": 0.5514916501714594, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6864670791270086, + "recall": 0.6808045834879024, + "f1-score": 0.6187439049668, + "support": 24261.0 + }, + "cross-entropy": 0.6384330519984234, + "roc-auc": 0.6413971749638401, + "score": 0.6413971749638401 + }, + "test": { + "0": { + "precision": 0.659637469489016, + "recall": 0.9741756293582319, + "f1-score": 0.7866294467110443, + "support": 16922.0 + }, + "1": { + "precision": 0.7015027322404371, + "recall": 0.1077310395468373, + "f1-score": 0.18677821223970173, + "support": 9533.0 + }, + "accuracy": 0.661954261954262, + "macro avg": { + "precision": 0.6805701008647266, + "recall": 0.5409533344525346, + "f1-score": 0.486703829475373, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6747235231578611, + "recall": 0.661954261954262, + "f1-score": 0.5704743978274568, + "support": 26455.0 + }, + "cross-entropy": 0.6791357747346614, + "roc-auc": 0.6040305589800323, + "score": 0.6040305589800323 + } + }, + "time": "0:01:00.788883", + "diversity": { + "train": { + "mean_centered_corr": -0.013087093830108643, + "mean_pairwise_disagreement": 0.025822423398123243, + "member_std": 0.02709772437810898 + }, + "val": { + "mean_centered_corr": -0.004288136959075928, + "mean_pairwise_disagreement": 0.08692423855623854, + "member_std": 0.060995232313871384 + }, + "test": { + "mean_centered_corr": 0.0036114430986344814, + "mean_pairwise_disagreement": 0.05469726437468373, + "member_std": 0.05942297354340553 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 14, + 0, + 25, + 25, + 0, + 25, + 0, + 25, + 0 + ], + "head_selection_time": "0:00:48.737044" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/1/report.json new file mode 100644 index 000000000..376991fac --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/1/report.json @@ -0,0 +1,223 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 428, + "metrics": { + "train": { + "0": { + "precision": 0.7932819825198438, + "recall": 0.9628529680799249, + "f1-score": 0.8698806559629089, + "support": 84179.0 + }, + "1": { + "precision": 0.5637555803571429, + "recall": 0.1605993164295366, + "f1-score": 0.24998453448809155, + "support": 25162.0 + }, + "accuracy": 0.7782350627852315, + "macro avg": { + "precision": 0.6785187814384934, + "recall": 0.5617261422547307, + "f1-score": 0.5599325952255002, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7404624241545658, + "recall": 0.7782350627852315, + "f1-score": 0.7272276144821345, + "support": 109341.0 + }, + "cross-entropy": 0.4937799085537247, + "roc-auc": 0.6959439172205661, + "score": 0.6959439172205661 + }, + "val": { + "0": { + "precision": 0.6909425020552251, + "recall": 0.9207372084031448, + "f1-score": 0.7894576898638008, + "support": 15518.0 + }, + "1": { + "precision": 0.6566164154103853, + "recall": 0.2690152121697358, + "f1-score": 0.3816632860040568, + "support": 8743.0 + }, + "accuracy": 0.6858744487036808, + "macro avg": { + "precision": 0.6737794587328052, + "recall": 0.5948762102864402, + "f1-score": 0.5855604879339288, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6785723204660147, + "recall": 0.6858744487036808, + "f1-score": 0.642499754372859, + "support": 24261.0 + }, + "cross-entropy": 0.654021472689771, + "roc-auc": 0.6327357100453991, + "score": 0.6327357100453991 + }, + "test": { + "0": { + "precision": 0.6651265486363822, + "recall": 0.9612929913721783, + "f1-score": 0.7862442301650596, + "support": 16922.0 + }, + "1": { + "precision": 0.6721721721721722, + "recall": 0.14087905171509493, + "f1-score": 0.2329372994536467, + "support": 9533.0 + }, + "accuracy": 0.6656586656586656, + "macro avg": { + "precision": 0.6686493604042771, + "recall": 0.5510860215436366, + "f1-score": 0.5095907648093532, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6676654232977576, + "recall": 0.6656586656586656, + "f1-score": 0.5868613168983084, + "support": 26455.0 + }, + "cross-entropy": 0.7032628295376716, + "roc-auc": 0.5971884680332056, + "score": 0.5971884680332056 + } + }, + "time": "0:01:04.856616", + "diversity": { + "train": { + "mean_centered_corr": -0.009633250534534454, + "mean_pairwise_disagreement": 0.02490320677749485, + "member_std": 0.024462362751364708 + }, + "val": { + "mean_centered_corr": 0.023074615746736526, + "mean_pairwise_disagreement": 0.06982341565049974, + "member_std": 0.0468476265668869 + }, + "test": { + "mean_centered_corr": 0.04183991998434067, + "mean_pairwise_disagreement": 0.04013975344620506, + "member_std": 0.043004557490348816 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 15, + 10, + 9, + 20, + 9 + ], + "head_selection_time": "0:00:29.837541" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/2/report.json new file mode 100644 index 000000000..eb29e4009 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/2/report.json @@ -0,0 +1,220 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7869066099388643, + "recall": 0.9755402178690648, + "f1-score": 0.8711287439600714, + "support": 84179.0 + }, + "1": { + "precision": 0.5867951033513947, + "recall": 0.11620697877752166, + "f1-score": 0.19399568751036655, + "support": 25162.0 + }, + "accuracy": 0.7777869234779269, + "macro avg": { + "precision": 0.6868508566451295, + "recall": 0.5458735983232932, + "f1-score": 0.532562215735219, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7408561281547767, + "recall": 0.7777869234779269, + "f1-score": 0.7153041039221398, + "support": 109341.0 + }, + "cross-entropy": 0.4984563455304167, + "roc-auc": 0.6822619490208846, + "score": 0.6822619490208846 + }, + "val": { + "0": { + "precision": 0.6891333565632933, + "recall": 0.8921252738754994, + "f1-score": 0.7775999101300306, + "support": 15518.0 + }, + "1": { + "precision": 0.5987535953978907, + "recall": 0.2857142857142857, + "f1-score": 0.3868370112272551, + "support": 8743.0 + }, + "accuracy": 0.673591360619925, + "macro avg": { + "precision": 0.643943475980592, + "recall": 0.5889197797948926, + "f1-score": 0.5822184606786429, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6565629657356641, + "recall": 0.673591360619925, + "f1-score": 0.6367796626090312, + "support": 24261.0 + }, + "cross-entropy": 0.6333720055734767, + "roc-auc": 0.6356383175142475, + "score": 0.6356383175142475 + }, + "test": { + "0": { + "precision": 0.6669322375041932, + "recall": 0.9399007209549699, + "f1-score": 0.7802305616875154, + "support": 16922.0 + }, + "1": { + "precision": 0.6098964326812428, + "recall": 0.16678904856813176, + "f1-score": 0.26194398682042835, + "support": 9533.0 + }, + "accuracy": 0.6613116613116613, + "macro avg": { + "precision": 0.638414335092718, + "recall": 0.5533448847615509, + "f1-score": 0.5210872742539718, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.646379512976611, + "recall": 0.6613116613116613, + "f1-score": 0.5934671552158488, + "support": 26455.0 + }, + "cross-entropy": 0.662735308091, + "roc-auc": 0.5998927728985708, + "score": 0.5998927728985708 + } + }, + "time": "0:01:00.829886", + "diversity": { + "train": { + "mean_centered_corr": -0.0152942705899477, + "mean_pairwise_disagreement": 0.023205388676030093, + "member_std": 0.024158649146556854 + }, + "val": { + "mean_centered_corr": -0.00040229607839137316, + "mean_pairwise_disagreement": 0.06998355252223468, + "member_std": 0.05039098858833313 + }, + "test": { + "mean_centered_corr": 0.0061957864090800285, + "mean_pairwise_disagreement": 0.04117994037348876, + "member_std": 0.047571223229169846 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31, + 20 + ], + "head_selection_time": "0:00:15.755507" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..72357a786 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-best-head-evaluation/0/report.json @@ -0,0 +1,390 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.784937732846757, + "recall": 0.978628874184773, + "f1-score": 0.8711468302225982, + "support": 84179.0 + }, + "1": { + "precision": 0.5902050113895216, + "recall": 0.10297273666640172, + "f1-score": 0.17535192203573363, + "support": 25162.0 + }, + "accuracy": 0.7771192873670444, + "macro avg": { + "precision": 0.6875713721181393, + "recall": 0.5408008054255873, + "f1-score": 0.5232493761291659, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.740125039188322, + "recall": 0.7771192873670444, + "f1-score": 0.7110276482158681, + "support": 109341.0 + }, + "cross-entropy": 0.4989568858959861, + "roc-auc": 0.6810795630080747, + "score": 0.6810795630080747 + }, + "val": { + "0": { + "precision": 0.6882539682539682, + "recall": 0.9081067147828329, + "f1-score": 0.783041146889673, + "support": 15518.0 + }, + "1": { + "precision": 0.6233491811938722, + "recall": 0.26993022989820425, + "f1-score": 0.3767259956899992, + "support": 8743.0 + }, + "accuracy": 0.6781253864226536, + "macro avg": { + "precision": 0.6558015747239202, + "recall": 0.5890184723405185, + "f1-score": 0.579883571289836, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6648640604485843, + "recall": 0.6781253864226536, + "f1-score": 0.6366162935473232, + "support": 24261.0 + }, + "cross-entropy": 0.6298773246042463, + "roc-auc": 0.6410202011331968, + "score": 0.6410202011331968 + }, + "test": { + "0": { + "precision": 0.6683168316831684, + "recall": 0.9413780876964898, + "f1-score": 0.781687030766966, + "support": 16922.0 + }, + "1": { + "precision": 0.62122947689958, + "recall": 0.17067030315745305, + "f1-score": 0.26777485187623434, + "support": 9533.0 + }, + "accuracy": 0.6636552636552636, + "macro avg": { + "precision": 0.6447731542913742, + "recall": 0.5560241954269715, + "f1-score": 0.5247309413216001, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6513490088462018, + "recall": 0.6636552636552636, + "f1-score": 0.5964999280882533, + "support": 26455.0 + }, + "cross-entropy": 0.6607089123106477, + "roc-auc": 0.6024701571918213, + "score": 0.6024701571918213 + } + }, + "time": "0:01:11.748716", + "diversity": { + "train": { + "mean_centered_corr": -0.014211982488632202, + "mean_pairwise_disagreement": 0.02565772703979353, + "member_std": 0.02677466720342636 + }, + "val": { + "mean_centered_corr": -0.005956840701401234, + "mean_pairwise_disagreement": 0.08571943089333604, + "member_std": 0.0601416639983654 + }, + "test": { + "mean_centered_corr": 0.00013841522741131485, + "mean_pairwise_disagreement": 0.05374807189323318, + "member_std": 0.05852190777659416 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0 + ], + "head_selection_time": "0:00:04.721148" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-best-head-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-best-head-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-best-head-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-best-head-evaluation/1/report.json new file mode 100644 index 000000000..b203d8213 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-best-head-evaluation/1/report.json @@ -0,0 +1,390 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 428, + "metrics": { + "train": { + "0": { + "precision": 0.792594913777064, + "recall": 0.9615105905273287, + "f1-score": 0.8689196878120001, + "support": 84179.0 + }, + "1": { + "precision": 0.5513708114095819, + "recall": 0.15825451077020905, + "f1-score": 0.24592391304347827, + "support": 25162.0 + }, + "accuracy": 0.7766620023595906, + "macro avg": { + "precision": 0.671982862593323, + "recall": 0.5598825506487689, + "f1-score": 0.5574218004277391, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7370834325964402, + "recall": 0.7766620023595906, + "f1-score": 0.7255533413845342, + "support": 109341.0 + }, + "cross-entropy": 0.49701600203519025, + "roc-auc": 0.6913539415681078, + "score": 0.6913539415681078 + }, + "val": { + "0": { + "precision": 0.6963472629768289, + "recall": 0.8869699703570048, + "f1-score": 0.7801836526470922, + "support": 15518.0 + }, + "1": { + "precision": 0.6097886540600668, + "recall": 0.3135079492165161, + "f1-score": 0.41411089288412145, + "support": 8743.0 + }, + "accuracy": 0.6803099624912411, + "macro avg": { + "precision": 0.6530679585184478, + "recall": 0.6002389597867605, + "f1-score": 0.5971472727656069, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6651539107753842, + "recall": 0.6803099624912411, + "f1-score": 0.6482610551198817, + "support": 24261.0 + }, + "cross-entropy": 0.6721133601344941, + "roc-auc": 0.6322301668779651, + "score": 0.6322301668779651 + }, + "test": { + "0": { + "precision": 0.6682340213049268, + "recall": 0.9490013000827325, + "f1-score": 0.7842457391219417, + "support": 16922.0 + }, + "1": { + "precision": 0.6438299628559637, + "recall": 0.16364208538760097, + "f1-score": 0.2609568417530947, + "support": 9533.0 + }, + "accuracy": 0.665998865998866, + "macro avg": { + "precision": 0.6560319920804453, + "recall": 0.5563216927351667, + "f1-score": 0.5226012904375182, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6594400734994472, + "recall": 0.665998865998866, + "f1-score": 0.5956797569402287, + "support": 26455.0 + }, + "cross-entropy": 0.7354332507178938, + "roc-auc": 0.5946009422441442, + "score": 0.5946009422441442 + } + }, + "time": "0:01:16.578012", + "diversity": { + "train": { + "mean_centered_corr": -0.006815396714955568, + "mean_pairwise_disagreement": 0.024371262617009646, + "member_std": 0.024308200925588608 + }, + "val": { + "mean_centered_corr": 0.023836545646190643, + "mean_pairwise_disagreement": 0.06752075214302525, + "member_std": 0.04681727662682533 + }, + "test": { + "mean_centered_corr": 0.04357389360666275, + "mean_pairwise_disagreement": 0.038735893574603254, + "member_std": 0.043050214648246765 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 10 + ], + "head_selection_time": "0:00:04.824193" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-best-head-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-best-head-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-best-head-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-best-head-evaluation/2/report.json new file mode 100644 index 000000000..e54db3a3d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-best-head-evaluation/2/report.json @@ -0,0 +1,390 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7854278840646122, + "recall": 0.9773458938690173, + "f1-score": 0.8709395692516158, + "support": 84179.0 + }, + "1": { + "precision": 0.5848029610276507, + "recall": 0.1067482712026071, + "f1-score": 0.1805410855318434, + "support": 25162.0 + }, + "accuracy": 0.7770003932651064, + "macro avg": { + "precision": 0.6851154225461314, + "recall": 0.5420470825358121, + "f1-score": 0.5257403273917296, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7392592527784887, + "recall": 0.7770003932651064, + "f1-score": 0.7120622346071831, + "support": 109341.0 + }, + "cross-entropy": 0.4995940703011633, + "roc-auc": 0.6793517558838738, + "score": 0.6793517558838738 + }, + "val": { + "0": { + "precision": 0.6817984256531607, + "recall": 0.9097821884263436, + "f1-score": 0.7794616977225672, + "support": 15518.0 + }, + "1": { + "precision": 0.6060776589758019, + "recall": 0.24636852339014068, + "f1-score": 0.350329348621615, + "support": 8743.0 + }, + "accuracy": 0.670706071472734, + "macro avg": { + "precision": 0.6439380423144814, + "recall": 0.5780753559082421, + "f1-score": 0.5648955231720911, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6545107349948965, + "recall": 0.670706071472734, + "f1-score": 0.6248141511173314, + "support": 24261.0 + }, + "cross-entropy": 0.6449115869812975, + "roc-auc": 0.6333770862914994, + "score": 0.6333770862914994 + }, + "test": { + "0": { + "precision": 0.6628663819907964, + "recall": 0.9448646732064768, + "f1-score": 0.7791340788928685, + "support": 16922.0 + }, + "1": { + "precision": 0.6002570694087404, + "recall": 0.1469631805307878, + "f1-score": 0.2361169630066571, + "support": 9533.0 + }, + "accuracy": 0.6573426573426573, + "macro avg": { + "precision": 0.6315617256997683, + "recall": 0.5459139268686323, + "f1-score": 0.5076255209497628, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6403052564249395, + "recall": 0.6573426573426573, + "f1-score": 0.583459077352772, + "support": 26455.0 + }, + "cross-entropy": 0.6839435417360092, + "roc-auc": 0.5972389647476771, + "score": 0.5972389647476771 + } + }, + "time": "0:01:12.365895", + "diversity": { + "train": { + "mean_centered_corr": -0.01532948762178421, + "mean_pairwise_disagreement": 0.02282787040011848, + "member_std": 0.0240154005587101 + }, + "val": { + "mean_centered_corr": 0.0001475554599892348, + "mean_pairwise_disagreement": 0.07275033872230886, + "member_std": 0.05133147910237312 + }, + "test": { + "mean_centered_corr": 0.007742947433143854, + "mean_pairwise_disagreement": 0.042625182141311176, + "member_std": 0.048414889723062515 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 20 + ], + "head_selection_time": "0:00:04.822432" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/0.toml new file mode 100644 index 000000000..0e6ac6970 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/0.toml @@ -0,0 +1,44 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 + +[cf_fisd] +lambda = 0.05 +variant = "raw" +dataset_name = "ecom-offers" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/0/report.json new file mode 100644 index 000000000..0ed76a5a7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/0/report.json @@ -0,0 +1,386 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7863951555903647, + "recall": 0.97808241960584, + "f1-score": 0.8718266385001879, + "support": 84179.0 + }, + "1": { + "precision": 0.602627611458109, + "recall": 0.11119942770844925, + "f1-score": 0.18775373259520214, + "support": 25162.0 + }, + "accuracy": 0.7785917450910455, + "macro avg": { + "precision": 0.6945113835242368, + "recall": 0.5446409236571446, + "f1-score": 0.5297901855476951, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7441058135735931, + "recall": 0.7785917450910455, + "f1-score": 0.7144049718025973, + "support": 109341.0 + }, + "cross-entropy": 0.49233822908120295, + "roc-auc": 0.6988844522375441, + "score": 0.6988844522375441 + }, + "val": { + "0": { + "precision": 0.6753571753571753, + "recall": 0.9565021265627014, + "f1-score": 0.7917111158523575, + "support": 15518.0 + }, + "1": { + "precision": 0.7043363994743759, + "recall": 0.1839185634221663, + "f1-score": 0.2916742245601306, + "support": 8743.0 + }, + "accuracy": 0.6780841680062653, + "macro avg": { + "precision": 0.6898467874157757, + "recall": 0.5702103449924338, + "f1-score": 0.5416926702062441, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6858004941180131, + "recall": 0.6780841680062653, + "f1-score": 0.6115115139988503, + "support": 24261.0 + }, + "cross-entropy": 0.6866508172905287, + "roc-auc": 0.6228186386127663, + "score": 0.6228186386127663 + }, + "test": { + "0": { + "precision": 0.6564383127180463, + "recall": 0.97848954024347, + "f1-score": 0.7857447919138234, + "support": 16922.0 + }, + "1": { + "precision": 0.7043054427294883, + "recall": 0.09094723591733976, + "f1-score": 0.16109253065774806, + "support": 9533.0 + }, + "accuracy": 0.6586656586656586, + "macro avg": { + "precision": 0.6803718777237673, + "recall": 0.5347183880804048, + "f1-score": 0.47341866128578575, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6736871258119445, + "recall": 0.6586656586656586, + "f1-score": 0.5606527485740326, + "support": 26455.0 + }, + "cross-entropy": 0.7554621625639649, + "roc-auc": 0.5911736900637132, + "score": 0.5911736900637132 + } + }, + "time": "0:01:11.748716", + "diversity": { + "train": { + "mean_centered_corr": -0.014211982488632202, + "mean_pairwise_disagreement": 0.02565772703979353, + "member_std": 0.02677466720342636 + }, + "val": { + "mean_centered_corr": -0.005956840701401234, + "mean_pairwise_disagreement": 0.08571943089333604, + "member_std": 0.0601416639983654 + }, + "test": { + "mean_centered_corr": 0.00013841522741131485, + "mean_pairwise_disagreement": 0.05374807189323318, + "member_std": 0.05852190777659416 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/1.toml new file mode 100644 index 000000000..02ae43937 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/1.toml @@ -0,0 +1,44 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 1 +head_selection = true + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 + +[cf_fisd] +lambda = 0.05 +variant = "raw" +dataset_name = "ecom-offers" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/1/report.json new file mode 100644 index 000000000..37db253a4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/1/report.json @@ -0,0 +1,386 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 428, + "metrics": { + "train": { + "0": { + "precision": 0.7883172407167219, + "recall": 0.9747324154480334, + "f1-score": 0.8716695703905154, + "support": 84179.0 + }, + "1": { + "precision": 0.5953196347031964, + "recall": 0.12435418488196487, + "f1-score": 0.20573344730094023, + "support": 25162.0 + }, + "accuracy": 0.7790398843983501, + "macro avg": { + "precision": 0.6918184377099592, + "recall": 0.5495433001649991, + "f1-score": 0.5387015088457279, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7439038389505744, + "recall": 0.7790398843983501, + "f1-score": 0.7184216146449133, + "support": 109341.0 + }, + "cross-entropy": 0.4903796076721471, + "roc-auc": 0.7031730821629574, + "score": 0.7031730821629574 + }, + "val": { + "0": { + "precision": 0.6731283119706508, + "recall": 0.9577265111483438, + "f1-score": 0.7905949942814586, + "support": 15518.0 + }, + "1": { + "precision": 0.6993583868011, + "recall": 0.1745396317053643, + "f1-score": 0.2793592677345538, + "support": 8743.0 + }, + "accuracy": 0.6754874077737933, + "macro avg": { + "precision": 0.6862433493858754, + "recall": 0.5661330714268541, + "f1-score": 0.5349771310080063, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6825809126154146, + "recall": 0.6754874077737933, + "f1-score": 0.6063596388880459, + "support": 24261.0 + }, + "cross-entropy": 0.7146728501800901, + "roc-auc": 0.6202479078617598, + "score": 0.6202479078617598 + }, + "test": { + "0": { + "precision": 0.6551465063861758, + "recall": 0.979080486940078, + "f1-score": 0.7850086470351331, + "support": 16922.0 + }, + "1": { + "precision": 0.6963979416809606, + "recall": 0.0851778034197, + "f1-score": 0.15178988690531825, + "support": 9533.0 + }, + "accuracy": 0.656964656964657, + "macro avg": { + "precision": 0.6757722240335682, + "recall": 0.532129145179889, + "f1-score": 0.46839926697022566, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.670011368705782, + "recall": 0.656964656964657, + "f1-score": 0.5568296850121686, + "support": 26455.0 + }, + "cross-entropy": 0.7991918340623438, + "roc-auc": 0.5895844228260869, + "score": 0.5895844228260869 + } + }, + "time": "0:01:16.578012", + "diversity": { + "train": { + "mean_centered_corr": -0.006815396714955568, + "mean_pairwise_disagreement": 0.024371262617009646, + "member_std": 0.024308200925588608 + }, + "val": { + "mean_centered_corr": 0.023836545646190643, + "mean_pairwise_disagreement": 0.06752075214302525, + "member_std": 0.04681727662682533 + }, + "test": { + "mean_centered_corr": 0.04357389360666275, + "mean_pairwise_disagreement": 0.038735893574603254, + "member_std": 0.043050214648246765 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/2.toml new file mode 100644 index 000000000..abc9c53f7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/2.toml @@ -0,0 +1,44 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 2 +head_selection = true + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 + +[cf_fisd] +lambda = 0.05 +variant = "raw" +dataset_name = "ecom-offers" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/2/report.json new file mode 100644 index 000000000..044a1ff43 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/2/report.json @@ -0,0 +1,386 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7860148668377912, + "recall": 0.9785338386058281, + "f1-score": 0.871772076877487, + "support": 84179.0 + }, + "1": { + "precision": 0.6023327464788732, + "recall": 0.10877513711151737, + "f1-score": 0.18427253753450482, + "support": 25162.0 + }, + "accuracy": 0.7783813939876167, + "macro avg": { + "precision": 0.6941738066583323, + "recall": 0.5436544878586728, + "f1-score": 0.528022307205996, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7437451828905884, + "recall": 0.7783813939876167, + "f1-score": 0.7135618592194436, + "support": 109341.0 + }, + "cross-entropy": 0.491969658143707, + "roc-auc": 0.6998568314138789, + "score": 0.6998568314138789 + }, + "val": { + "0": { + "precision": 0.676559218336225, + "recall": 0.9548910942131718, + "f1-score": 0.791982896846606, + "support": 15518.0 + }, + "1": { + "precision": 0.7032640949554896, + "recall": 0.18975180144115292, + "f1-score": 0.2988650693568726, + "support": 8743.0 + }, + "accuracy": 0.6791558468323647, + "macro avg": { + "precision": 0.6899116566458573, + "recall": 0.5723214478271623, + "f1-score": 0.5454239831017393, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6861829245429861, + "recall": 0.6791558468323647, + "f1-score": 0.614276736105386, + "support": 24261.0 + }, + "cross-entropy": 0.7019446862265161, + "roc-auc": 0.6211436845976699, + "score": 0.6211436845976699 + }, + "test": { + "0": { + "precision": 0.65782767956681, + "recall": 0.9763621321356813, + "f1-score": 0.7860507160188401, + "support": 16922.0 + }, + "1": { + "precision": 0.7012696041822255, + "recall": 0.09849994755061366, + "f1-score": 0.1727373068432671, + "support": 9533.0 + }, + "accuracy": 0.66002646002646, + "macro avg": { + "precision": 0.6795486418745178, + "recall": 0.5374310398431476, + "f1-score": 0.47939401143105365, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6734818798071712, + "recall": 0.66002646002646, + "f1-score": 0.5650446026311728, + "support": 26455.0 + }, + "cross-entropy": 0.7793654776091676, + "roc-auc": 0.5891830464738508, + "score": 0.5891830464738508 + } + }, + "time": "0:01:12.365895", + "diversity": { + "train": { + "mean_centered_corr": -0.01532948762178421, + "mean_pairwise_disagreement": 0.02282787040011848, + "member_std": 0.0240154005587101 + }, + "val": { + "mean_centered_corr": 0.0001475554599892348, + "mean_pairwise_disagreement": 0.07275033872230886, + "member_std": 0.05133147910237312 + }, + "test": { + "mean_centered_corr": 0.007742947433143854, + "mean_pairwise_disagreement": 0.042625182141311176, + "member_std": 0.048414889723062515 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..edf46fa0f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,391 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7827118483860296, + "recall": 0.9837013982109553, + "f1-score": 0.8717719266847043, + "support": 84179.0 + }, + "1": { + "precision": 0.6130851663846588, + "recall": 0.08640012717590016, + "f1-score": 0.15145604012818725, + "support": 25162.0 + }, + "accuracy": 0.7772107443685351, + "macro avg": { + "precision": 0.6978985073853442, + "recall": 0.5350507626934277, + "f1-score": 0.5116139834064457, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7436766596414737, + "recall": 0.7772107443685351, + "f1-score": 0.7060098764241883, + "support": 109341.0 + }, + "cross-entropy": 0.4979833881789, + "roc-auc": 0.6847415093580901, + "score": 0.6847415093580901 + }, + "val": { + "0": { + "precision": 0.6773795373502789, + "recall": 0.9548266529191907, + "f1-score": 0.7925224646983312, + "support": 15518.0 + }, + "1": { + "precision": 0.7063259321323837, + "recall": 0.19283998627473406, + "f1-score": 0.30296495956873315, + "support": 8743.0 + }, + "accuracy": 0.6802275256584642, + "macro avg": { + "precision": 0.6918527347413312, + "recall": 0.5738333195969624, + "f1-score": 0.5477437121335322, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6878110253177965, + "recall": 0.6802275256584642, + "f1-score": 0.6160993466344395, + "support": 24261.0 + }, + "cross-entropy": 0.6392793150076335, + "roc-auc": 0.6419149054445072, + "score": 0.6419149054445072 + }, + "test": { + "0": { + "precision": 0.658847933620552, + "recall": 0.9760075641177166, + "f1-score": 0.7866634913074542, + "support": 16922.0 + }, + "1": { + "precision": 0.7072819033886085, + "recall": 0.10290569600335676, + "f1-score": 0.17967032967032967, + "support": 9533.0 + }, + "accuracy": 0.6613872613872613, + "macro avg": { + "precision": 0.6830649185045803, + "recall": 0.5394566300605367, + "f1-score": 0.48316691048889193, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6763010061512224, + "recall": 0.6613872613872613, + "f1-score": 0.5679348649651103, + "support": 26455.0 + }, + "cross-entropy": 0.6803501878150763, + "roc-auc": 0.6041224275423289, + "score": 0.6041224275423289 + } + }, + "time": "0:01:11.748716", + "diversity": { + "train": { + "mean_centered_corr": -0.014211982488632202, + "mean_pairwise_disagreement": 0.02565772703979353, + "member_std": 0.02677466720342636 + }, + "val": { + "mean_centered_corr": -0.005956840701401234, + "mean_pairwise_disagreement": 0.08571943089333604, + "member_std": 0.0601416639983654 + }, + "test": { + "mean_centered_corr": 0.00013841522741131485, + "mean_pairwise_disagreement": 0.05374807189323318, + "member_std": 0.05852190777659416 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0, + 25 + ], + "head_selection_time": "0:00:15.652537" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/1/report.json new file mode 100644 index 000000000..b23c33390 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/1/report.json @@ -0,0 +1,392 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 428, + "metrics": { + "train": { + "0": { + "precision": 0.7934533487433643, + "recall": 0.9623540312904644, + "f1-score": 0.8697799513627554, + "support": 84179.0 + }, + "1": { + "precision": 0.5624741129366285, + "recall": 0.16191081790000794, + "f1-score": 0.2514426785989816, + "support": 25162.0 + }, + "accuracy": 0.7781527514838898, + "macro avg": { + "precision": 0.6779637308399964, + "recall": 0.5621324245952362, + "f1-score": 0.5606113149808685, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7402994583329137, + "recall": 0.7781527514838898, + "f1-score": 0.7274856385497934, + "support": 109341.0 + }, + "cross-entropy": 0.49515974918562666, + "roc-auc": 0.6937247371184572, + "score": 0.6937247371184572 + }, + "val": { + "0": { + "precision": 0.6973154362416107, + "recall": 0.8704085578038407, + "f1-score": 0.7743063517541848, + "support": 15518.0 + }, + "1": { + "precision": 0.5888366387241872, + "recall": 0.3294063822486561, + "f1-score": 0.42247322869297343, + "support": 8743.0 + }, + "accuracy": 0.6754461893574049, + "macro avg": { + "precision": 0.643076037482899, + "recall": 0.5999074700262484, + "f1-score": 0.5983897902235791, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.65822264836416, + "recall": 0.6754461893574049, + "f1-score": 0.6475153293344919, + "support": 24261.0 + }, + "cross-entropy": 0.6446266565354483, + "roc-auc": 0.6333421643138163, + "score": 0.6333421643138163 + }, + "test": { + "0": { + "precision": 0.669557164505809, + "recall": 0.9399598156246307, + "f1-score": 0.7820443482963764, + "support": 16922.0 + }, + "1": { + "precision": 0.6235642830678029, + "recall": 0.1765446344277772, + "f1-score": 0.2751798561151079, + "support": 9533.0 + }, + "accuracy": 0.6648648648648648, + "macro avg": { + "precision": 0.646560723786806, + "recall": 0.558252225026204, + "f1-score": 0.5286121022057422, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6529837326876835, + "recall": 0.664864864864865, + "f1-score": 0.5993968637390514, + "support": 26455.0 + }, + "cross-entropy": 0.682979018710851, + "roc-auc": 0.5975759215250558, + "score": 0.5975759215250558 + } + }, + "time": "0:01:16.578012", + "diversity": { + "train": { + "mean_centered_corr": -0.006815396714955568, + "mean_pairwise_disagreement": 0.024371262617009646, + "member_std": 0.024308200925588608 + }, + "val": { + "mean_centered_corr": 0.023836545646190643, + "mean_pairwise_disagreement": 0.06752075214302525, + "member_std": 0.04681727662682533 + }, + "test": { + "mean_centered_corr": 0.04357389360666275, + "mean_pairwise_disagreement": 0.038735893574603254, + "member_std": 0.043050214648246765 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 10, + 9, + 11 + ], + "head_selection_time": "0:00:20.590968" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/2/report.json new file mode 100644 index 000000000..e79af5c4a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/2/report.json @@ -0,0 +1,392 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7875746118265743, + "recall": 0.9749462455006593, + "f1-score": 0.8713007935875998, + "support": 84179.0 + }, + "1": { + "precision": 0.5892891918208374, + "recall": 0.12026071059534219, + "f1-score": 0.1997557513945275, + "support": 25162.0 + }, + "accuracy": 0.7782624998856787, + "macro avg": { + "precision": 0.6884319018237058, + "recall": 0.5476034780480008, + "f1-score": 0.5355282724910637, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7419443565866886, + "recall": 0.7782624998856787, + "f1-score": 0.7167620903412231, + "support": 109341.0 + }, + "cross-entropy": 0.4966763535938117, + "roc-auc": 0.6873519813280431, + "score": 0.6873519813280431 + }, + "val": { + "0": { + "precision": 0.6921494686321563, + "recall": 0.9107488078360614, + "f1-score": 0.7865431171216295, + "support": 15518.0 + }, + "1": { + "precision": 0.6395106715252473, + "recall": 0.28102481985588473, + "f1-score": 0.3904648390941597, + "support": 8743.0 + }, + "accuracy": 0.6838135278842586, + "macro avg": { + "precision": 0.6658300700787019, + "recall": 0.5958868138459731, + "f1-score": 0.5885039781078946, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6731798876954388, + "recall": 0.6838135278842586, + "f1-score": 0.6438073525284895, + "support": 24261.0 + }, + "cross-entropy": 0.6379588554594992, + "roc-auc": 0.6340638176219543, + "score": 0.6340638176219543 + }, + "test": { + "0": { + "precision": 0.668311699083482, + "recall": 0.9523106015837371, + "f1-score": 0.7854270744486413, + "support": 16922.0 + }, + "1": { + "precision": 0.6554227156276686, + "recall": 0.16101961607049198, + "f1-score": 0.25852631578947366, + "support": 9533.0 + }, + "accuracy": 0.6671706671706672, + "macro avg": { + "precision": 0.6618672073555754, + "recall": 0.5566651088271146, + "f1-score": 0.5219766951190575, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6636671827620203, + "recall": 0.6671706671706672, + "f1-score": 0.5955595661402745, + "support": 26455.0 + }, + "cross-entropy": 0.6753783561951368, + "roc-auc": 0.598916340879379, + "score": 0.598916340879379 + } + }, + "time": "0:01:12.365895", + "diversity": { + "train": { + "mean_centered_corr": -0.01532948762178421, + "mean_pairwise_disagreement": 0.02282787040011848, + "member_std": 0.0240154005587101 + }, + "val": { + "mean_centered_corr": 0.0001475554599892348, + "mean_pairwise_disagreement": 0.07275033872230886, + "member_std": 0.05133147910237312 + }, + "test": { + "mean_centered_corr": 0.007742947433143854, + "mean_pairwise_disagreement": 0.042625182141311176, + "member_std": 0.048414889723062515 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 20, + 25, + 31 + ], + "head_selection_time": "0:00:20.602844" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-best-head-evaluation/0/report.json new file mode 100644 index 000000000..8fc29a25e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-best-head-evaluation/0/report.json @@ -0,0 +1,426 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.887499988079071, + "min_features_per_member": 92.0, + "mean_features_per_member": 97.625, + "max_features_per_member": 103.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.399999618530273, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.797659951359852, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7889205040357703, + "recall": 0.9683650316587272, + "f1-score": 0.8694808138449641, + "support": 84179.0 + }, + "1": { + "precision": 0.5572734829592685, + "recall": 0.13321675542484698, + "f1-score": 0.21503031080604293, + "support": 25162.0 + }, + "accuracy": 0.7761772802516896, + "macro avg": { + "precision": 0.6730969934975194, + "recall": 0.5507908935417871, + "f1-score": 0.5422555623255035, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7356129401363461, + "recall": 0.7761772802516896, + "f1-score": 0.7188759761585946, + "support": 109341.0 + }, + "cross-entropy": 0.4988956148214662, + "roc-auc": 0.68250001764071, + "score": 0.68250001764071 + }, + "val": { + "0": { + "precision": 0.6950127877237852, + "recall": 0.910619925248099, + "f1-score": 0.7883403068340307, + "support": 15518.0 + }, + "1": { + "precision": 0.6469839653855943, + "recall": 0.2907468832208624, + "f1-score": 0.401199494949495, + "support": 8743.0 + }, + "accuracy": 0.6872346564444994, + "macro avg": { + "precision": 0.6709983765546897, + "recall": 0.6006834042344806, + "f1-score": 0.5947699008917628, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6777045154471766, + "recall": 0.6872346564444994, + "f1-score": 0.6488253602816835, + "support": 24261.0 + }, + "cross-entropy": 0.6245548911838499, + "roc-auc": 0.6477890245840551, + "score": 0.6477890245840551 + }, + "test": { + "0": { + "precision": 0.6679904029122198, + "recall": 0.9542607256825434, + "f1-score": 0.7858672376873662, + "support": 16922.0 + }, + "1": { + "precision": 0.6606751424813678, + "recall": 0.1580824504353299, + "f1-score": 0.25512104283054005, + "support": 9533.0 + }, + "accuracy": 0.6673596673596673, + "macro avg": { + "precision": 0.6643327726967938, + "recall": 0.5561715880589366, + "f1-score": 0.5204941402589531, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6653543651996018, + "recall": 0.6673596673596673, + "f1-score": 0.5946140350576129, + "support": 26455.0 + }, + "cross-entropy": 0.6584188876218103, + "roc-auc": 0.6045088303107439, + "score": 0.6045088303107439 + } + }, + "time": "0:01:18.654365", + "diversity": { + "train": { + "mean_centered_corr": -0.01063462533056736, + "mean_pairwise_disagreement": 0.026665487313881315, + "member_std": 0.029800426214933395 + }, + "val": { + "mean_centered_corr": -0.013281173072755337, + "mean_pairwise_disagreement": 0.09412748922670262, + "member_std": 0.06749773025512695 + }, + "test": { + "mean_centered_corr": -0.011601177044212818, + "mean_pairwise_disagreement": 0.059119563958273634, + "member_std": 0.06550896167755127 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 22 + ], + "head_selection_time": "0:00:04.756683" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-best-head-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-best-head-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-best-head-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-best-head-evaluation/1/report.json new file mode 100644 index 000000000..990349219 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-best-head-evaluation/1/report.json @@ -0,0 +1,426 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + }, + "mask_stats": { + "keep_rate_actual": 0.9005681872367859, + "min_features_per_member": 89.0, + "mean_features_per_member": 99.0625, + "max_features_per_member": 106.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.81818199157715, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8186085273842617, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 428, + "metrics": { + "train": { + "0": { + "precision": 0.7888152627873106, + "recall": 0.9715249646586441, + "f1-score": 0.8706881866961929, + "support": 84179.0 + }, + "1": { + "precision": 0.5768008474576272, + "recall": 0.12983864557666322, + "f1-score": 0.21196392655550508, + "support": 25162.0 + }, + "accuracy": 0.7778326519786722, + "macro avg": { + "precision": 0.6828080551224689, + "recall": 0.5506818051176536, + "f1-score": 0.541326056625849, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.740025634756421, + "recall": 0.7778326519786722, + "f1-score": 0.7190998544725988, + "support": 109341.0 + }, + "cross-entropy": 0.497526106332481, + "roc-auc": 0.6852792162881653, + "score": 0.6852792162881653 + }, + "val": { + "0": { + "precision": 0.6946272468323348, + "recall": 0.9114576620698543, + "f1-score": 0.7884057971014493, + "support": 15518.0 + }, + "1": { + "precision": 0.6476019492177482, + "recall": 0.28880247054786684, + "f1-score": 0.39946211042556556, + "support": 8743.0 + }, + "accuracy": 0.6870697827789456, + "macro avg": { + "precision": 0.6711145980250415, + "recall": 0.6001300663088606, + "f1-score": 0.5939339537635074, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6776806173840709, + "recall": 0.6870697827789456, + "f1-score": 0.6482411438469564, + "support": 24261.0 + }, + "cross-entropy": 0.6172041286027814, + "roc-auc": 0.6512227217747169, + "score": 0.6512227217747169 + }, + "test": { + "0": { + "precision": 0.6737023929895517, + "recall": 0.9449828625457983, + "f1-score": 0.7866102612032072, + "support": 16922.0 + }, + "1": { + "precision": 0.6575947039352703, + "recall": 0.18755900555963495, + "f1-score": 0.29187071498530853, + "support": 9533.0 + }, + "accuracy": 0.6720468720468721, + "macro avg": { + "precision": 0.6656485484624111, + "recall": 0.5662709340527167, + "f1-score": 0.5392404880942578, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6678980233144632, + "recall": 0.6720468720468721, + "f1-score": 0.6083319737681201, + "support": 26455.0 + }, + "cross-entropy": 0.6427700166976323, + "roc-auc": 0.6075381341628896, + "score": 0.6075381341628896 + } + }, + "time": "0:01:23.222825", + "diversity": { + "train": { + "mean_centered_corr": -0.008371543139219284, + "mean_pairwise_disagreement": 0.02743249071932702, + "member_std": 0.027091743424534798 + }, + "val": { + "mean_centered_corr": 0.05517549440264702, + "mean_pairwise_disagreement": 0.07159946402762432, + "member_std": 0.054152194410562515 + }, + "test": { + "mean_centered_corr": 0.09162398427724838, + "mean_pairwise_disagreement": 0.04175463812560587, + "member_std": 0.052940964698791504 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 16 + ], + "head_selection_time": "0:00:04.823752" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-best-head-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-best-head-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-best-head-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-best-head-evaluation/2/report.json new file mode 100644 index 000000000..e1b081026 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-best-head-evaluation/2/report.json @@ -0,0 +1,426 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + }, + "mask_stats": { + "keep_rate_actual": 0.8940340876579285, + "min_features_per_member": 89.0, + "mean_features_per_member": 98.34375, + "max_features_per_member": 104.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.60909080505371, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8074834695307933, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.788166502634169, + "recall": 0.9703607788165695, + "f1-score": 0.8698254151647614, + "support": 84179.0 + }, + "1": { + "precision": 0.56251095914431, + "recall": 0.12749383991733568, + "f1-score": 0.20787299530212214, + "support": 25162.0 + }, + "accuracy": 0.7763967770552674, + "macro avg": { + "precision": 0.6753387308892396, + "recall": 0.5489273093669526, + "f1-score": 0.5388492052334418, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7362377221648864, + "recall": 0.7763967770552674, + "f1-score": 0.7174942055674125, + "support": 109341.0 + }, + "cross-entropy": 0.5009419379304306, + "roc-auc": 0.6745540773335442, + "score": 0.6745540773335442 + }, + "val": { + "0": { + "precision": 0.6869277961302501, + "recall": 0.9380074751901019, + "f1-score": 0.7930696305982348, + "support": 15518.0 + }, + "1": { + "precision": 0.6867469879518072, + "recall": 0.24122154866750542, + "f1-score": 0.3570340274250889, + "support": 8743.0 + }, + "accuracy": 0.6869049091133919, + "macro avg": { + "precision": 0.6868373920410287, + "recall": 0.5896145119288037, + "f1-score": 0.5750518290116619, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6868626378142645, + "recall": 0.6869049091133919, + "f1-score": 0.6359343402745542, + "support": 24261.0 + }, + "cross-entropy": 0.6325180503678897, + "roc-auc": 0.6430199007953441, + "score": 0.6430199007953441 + }, + "test": { + "0": { + "precision": 0.6649977612244067, + "recall": 0.965429618248434, + "f1-score": 0.7875340451685989, + "support": 16922.0 + }, + "1": { + "precision": 0.6901483050847458, + "recall": 0.13668310080772056, + "f1-score": 0.22817616671044566, + "support": 9533.0 + }, + "accuracy": 0.6667926667926668, + "macro avg": { + "precision": 0.6775730331545762, + "recall": 0.5510563595280773, + "f1-score": 0.5078551059395223, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.674060703376008, + "recall": 0.6667926667926668, + "f1-score": 0.5859706864333286, + "support": 26455.0 + }, + "cross-entropy": 0.6784012446842685, + "roc-auc": 0.5970664756329549, + "score": 0.5970664756329549 + } + }, + "time": "0:01:18.718791", + "diversity": { + "train": { + "mean_centered_corr": -0.007062356453388929, + "mean_pairwise_disagreement": 0.025052193183149135, + "member_std": 0.028095407411456108 + }, + "val": { + "mean_centered_corr": -0.003141294466331601, + "mean_pairwise_disagreement": 0.09142868017301098, + "member_std": 0.06040506064891815 + }, + "test": { + "mean_centered_corr": 0.00333536509424448, + "mean_pairwise_disagreement": 0.05701769895318282, + "member_std": 0.059019479900598526 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 23 + ], + "head_selection_time": "0:00:04.822535" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-evaluation/0.toml new file mode 100644 index 000000000..cd8d74155 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-evaluation/0.toml @@ -0,0 +1,57 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 47 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "ecom-offers" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-evaluation/0/report.json new file mode 100644 index 000000000..5e1e2aae3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-evaluation/0/report.json @@ -0,0 +1,422 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.887499988079071, + "min_features_per_member": 92.0, + "mean_features_per_member": 97.625, + "max_features_per_member": 103.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.399999618530273, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.797659951359852, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7842480378851866, + "recall": 0.9816818921583768, + "f1-score": 0.8719282511210762, + "support": 84179.0 + }, + "1": { + "precision": 0.6115869017632242, + "recall": 0.09649471425164931, + "f1-score": 0.1666895510091995, + "support": 25162.0 + }, + "accuracy": 0.7779789831810574, + "macro avg": { + "precision": 0.6979174698242054, + "recall": 0.539088303205013, + "f1-score": 0.5193089010651378, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7445145480954387, + "recall": 0.7779789831810574, + "f1-score": 0.7096358249294825, + "support": 109341.0 + }, + "cross-entropy": 0.4927771605769271, + "roc-auc": 0.6981357694476362, + "score": 0.6981357694476362 + }, + "val": { + "0": { + "precision": 0.6767607563716087, + "recall": 0.9548266529191907, + "f1-score": 0.7920987918314979, + "support": 15518.0 + }, + "1": { + "precision": 0.7038445289395859, + "recall": 0.19055244195356286, + "f1-score": 0.29990999099909993, + "support": 8743.0 + }, + "accuracy": 0.6794031573306953, + "macro avg": { + "precision": 0.6903026426555974, + "recall": 0.5726895474363768, + "f1-score": 0.5460043914152989, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6865210063020248, + "recall": 0.6794031573306953, + "f1-score": 0.6147274268557072, + "support": 24261.0 + }, + "cross-entropy": 0.673915591317614, + "roc-auc": 0.6250550271749447, + "score": 0.6250550271749447 + }, + "test": { + "0": { + "precision": 0.657534791252485, + "recall": 0.9772485521805933, + "f1-score": 0.7861285415478227, + "support": 16922.0 + }, + "1": { + "precision": 0.7049808429118773, + "recall": 0.09650687086961082, + "f1-score": 0.16977302085255583, + "support": 9533.0 + }, + "accuracy": 0.6598752598752599, + "macro avg": { + "precision": 0.6812578170821812, + "recall": 0.536877711525102, + "f1-score": 0.4779507812001893, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6746318697052912, + "recall": 0.6598752598752599, + "f1-score": 0.5640262100873057, + "support": 26455.0 + }, + "cross-entropy": 0.735759481729751, + "roc-auc": 0.592670158895295, + "score": 0.592670158895295 + } + }, + "time": "0:01:18.654365", + "diversity": { + "train": { + "mean_centered_corr": -0.01063462533056736, + "mean_pairwise_disagreement": 0.026665487313881315, + "member_std": 0.029800426214933395 + }, + "val": { + "mean_centered_corr": -0.013281173072755337, + "mean_pairwise_disagreement": 0.09412748922670262, + "member_std": 0.06749773025512695 + }, + "test": { + "mean_centered_corr": -0.011601177044212818, + "mean_pairwise_disagreement": 0.059119563958273634, + "member_std": 0.06550896167755127 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-evaluation/1.toml new file mode 100644 index 000000000..9e8dabb63 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-evaluation/1.toml @@ -0,0 +1,57 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 1 +head_selection = true + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 1 + +[bins] +n_bins = 47 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "ecom-offers" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-evaluation/1/report.json new file mode 100644 index 000000000..6da5b519a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-evaluation/1/report.json @@ -0,0 +1,422 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + }, + "mask_stats": { + "keep_rate_actual": 0.9005681872367859, + "min_features_per_member": 89.0, + "mean_features_per_member": 99.0625, + "max_features_per_member": 106.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.81818199157715, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8186085273842617, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 428, + "metrics": { + "train": { + "0": { + "precision": 0.7861034512370829, + "recall": 0.9787001508689815, + "f1-score": 0.8718925611963044, + "support": 84179.0 + }, + "1": { + "precision": 0.6048920229175848, + "recall": 0.10909307686193466, + "f1-score": 0.18484848484848485, + "support": 25162.0 + }, + "accuracy": 0.7785825993908964, + "macro avg": { + "precision": 0.6954977370773339, + "recall": 0.5438966138654581, + "f1-score": 0.5283705230223946, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7444023330894968, + "recall": 0.7785825993908964, + "f1-score": 0.7137871565533632, + "support": 109341.0 + }, + "cross-entropy": 0.49057469501963574, + "roc-auc": 0.7028790214614515, + "score": 0.7028790214614515 + }, + "val": { + "0": { + "precision": 0.6765809462798572, + "recall": 0.9528289728057739, + "f1-score": 0.7912875949909023, + "support": 15518.0 + }, + "1": { + "precision": 0.6958869962609057, + "recall": 0.1915818368980899, + "f1-score": 0.3004484304932735, + "support": 8743.0 + }, + "accuracy": 0.6784963521701496, + "macro avg": { + "precision": 0.6862339712703814, + "recall": 0.5722054048519319, + "f1-score": 0.5458680127420878, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6835383179868894, + "recall": 0.6784963521701496, + "f1-score": 0.6144026019896752, + "support": 24261.0 + }, + "cross-entropy": 0.6967356655889665, + "roc-auc": 0.6213201555665758, + "score": 0.6213201555665758 + }, + "test": { + "0": { + "precision": 0.6572702659659181, + "recall": 0.9755348067604303, + "f1-score": 0.7853846519815405, + "support": 16922.0 + }, + "1": { + "precision": 0.6908140403286034, + "recall": 0.09703136473303263, + "f1-score": 0.17016188373804267, + "support": 9533.0 + }, + "accuracy": 0.658968058968059, + "macro avg": { + "precision": 0.6740421531472607, + "recall": 0.5362830857467314, + "f1-score": 0.4777732678597916, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6693576899311223, + "recall": 0.658968058968059, + "f1-score": 0.5636905053300468, + "support": 26455.0 + }, + "cross-entropy": 0.7710291094729679, + "roc-auc": 0.5900624771932574, + "score": 0.5900624771932574 + } + }, + "time": "0:01:23.222825", + "diversity": { + "train": { + "mean_centered_corr": -0.008371543139219284, + "mean_pairwise_disagreement": 0.02743249071932702, + "member_std": 0.027091743424534798 + }, + "val": { + "mean_centered_corr": 0.05517549440264702, + "mean_pairwise_disagreement": 0.07159946402762432, + "member_std": 0.054152194410562515 + }, + "test": { + "mean_centered_corr": 0.09162398427724838, + "mean_pairwise_disagreement": 0.04175463812560587, + "member_std": 0.052940964698791504 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-evaluation/2.toml new file mode 100644 index 000000000..9551165d5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-evaluation/2.toml @@ -0,0 +1,57 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 2 +head_selection = true + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 2 + +[bins] +n_bins = 47 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "ecom-offers" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-evaluation/2/report.json new file mode 100644 index 000000000..254ffb5e5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-evaluation/2/report.json @@ -0,0 +1,422 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + }, + "mask_stats": { + "keep_rate_actual": 0.8940340876579285, + "min_features_per_member": 89.0, + "mean_features_per_member": 98.34375, + "max_features_per_member": 104.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.60909080505371, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8074834695307933, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7848450275717818, + "recall": 0.9806483802373513, + "f1-score": 0.8718888460543202, + "support": 84179.0 + }, + "1": { + "precision": 0.608507570295602, + "recall": 0.10062793100707416, + "f1-score": 0.17269720015005285, + "support": 25162.0 + }, + "accuracy": 0.7781344600835917, + "macro avg": { + "precision": 0.6966762989336919, + "recall": 0.5406381556222127, + "f1-score": 0.5222930231021865, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7442655276588193, + "recall": 0.7781344600835917, + "f1-score": 0.7109879928131465, + "support": 109341.0 + }, + "cross-entropy": 0.4923260014405221, + "roc-auc": 0.699065450692943, + "score": 0.699065450692943 + }, + "val": { + "0": { + "precision": 0.678972564905174, + "recall": 0.9505090862224513, + "f1-score": 0.7921164276891681, + "support": 15518.0 + }, + "1": { + "precision": 0.6972802522664564, + "recall": 0.20233329520759466, + "f1-score": 0.31365248226950354, + "support": 8743.0 + }, + "accuracy": 0.6808870203206793, + "macro avg": { + "precision": 0.6881264085858152, + "recall": 0.576421190715023, + "f1-score": 0.5528844549793358, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6855701540647177, + "recall": 0.6808870203206793, + "f1-score": 0.6196911247418813, + "support": 24261.0 + }, + "cross-entropy": 0.6857647009535869, + "roc-auc": 0.6208476032754839, + "score": 0.6208476032754839 + }, + "test": { + "0": { + "precision": 0.6596955128205129, + "recall": 0.9730528306346767, + "f1-score": 0.7863043789694857, + "support": 16922.0 + }, + "1": { + "precision": 0.6949832775919732, + "recall": 0.10898982481904962, + "f1-score": 0.18842945230322813, + "support": 9533.0 + }, + "accuracy": 0.6616896616896617, + "macro avg": { + "precision": 0.677339395206243, + "recall": 0.5410213277268632, + "f1-score": 0.4873669156363569, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6724113798235871, + "recall": 0.6616896616896617, + "f1-score": 0.5708614881772184, + "support": 26455.0 + }, + "cross-entropy": 0.7527583302993925, + "roc-auc": 0.5890089208341323, + "score": 0.5890089208341323 + } + }, + "time": "0:01:18.718791", + "diversity": { + "train": { + "mean_centered_corr": -0.007062356453388929, + "mean_pairwise_disagreement": 0.025052193183149135, + "member_std": 0.028095407411456108 + }, + "val": { + "mean_centered_corr": -0.003141294466331601, + "mean_pairwise_disagreement": 0.09142868017301098, + "member_std": 0.06040506064891815 + }, + "test": { + "mean_centered_corr": 0.00333536509424448, + "mean_pairwise_disagreement": 0.05701769895318282, + "member_std": 0.059019479900598526 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..5f7214ce5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/0/report.json @@ -0,0 +1,428 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.887499988079071, + "min_features_per_member": 92.0, + "mean_features_per_member": 97.625, + "max_features_per_member": 103.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.399999618530273, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.797659951359852, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7909659567005086, + "recall": 0.9643735373430428, + "f1-score": 0.8691043985161632, + "support": 84179.0 + }, + "1": { + "precision": 0.5528552258834054, + "recall": 0.14736507431841667, + "f1-score": 0.23270262637672973, + "support": 25162.0 + }, + "accuracy": 0.7763601942546712, + "macro avg": { + "precision": 0.671910591291957, + "recall": 0.5558693058307297, + "f1-score": 0.5509035124464464, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7361709373681451, + "recall": 0.7763601942546712, + "f1-score": 0.7226530089132472, + "support": 109341.0 + }, + "cross-entropy": 0.5000394688239905, + "roc-auc": 0.6809790060024956, + "score": 0.6809790060024956 + }, + "val": { + "0": { + "precision": 0.7075808427894092, + "recall": 0.8559092666580745, + "f1-score": 0.7747090903788387, + "support": 15518.0 + }, + "1": { + "precision": 0.5927140255009108, + "recall": 0.3721834610545579, + "f1-score": 0.4572472423241762, + "support": 8743.0 + }, + "accuracy": 0.6815877333992828, + "macro avg": { + "precision": 0.6501474341451601, + "recall": 0.6140463638563163, + "f1-score": 0.6159781663515075, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6661859875256797, + "recall": 0.6815877333992828, + "f1-score": 0.6603045341964096, + "support": 24261.0 + }, + "cross-entropy": 0.6203779736299148, + "roc-auc": 0.6506783465179156, + "score": 0.6506783465179156 + }, + "test": { + "0": { + "precision": 0.6800559465011583, + "recall": 0.9194539652523342, + "f1-score": 0.7818396522700435, + "support": 16922.0 + }, + "1": { + "precision": 0.618847874720358, + "recall": 0.23214098395048777, + "f1-score": 0.3376306354412999, + "support": 9533.0 + }, + "accuracy": 0.6717822717822718, + "macro avg": { + "precision": 0.6494519106107581, + "recall": 0.575797474601411, + "f1-score": 0.5597351438556717, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6579997549197419, + "recall": 0.6717822717822718, + "f1-score": 0.6217699279295251, + "support": 26455.0 + }, + "cross-entropy": 0.6443009499315193, + "roc-auc": 0.6065618416202598, + "score": 0.6065618416202598 + } + }, + "time": "0:01:18.654365", + "diversity": { + "train": { + "mean_centered_corr": -0.01063462533056736, + "mean_pairwise_disagreement": 0.026665487313881315, + "member_std": 0.029800426214933395 + }, + "val": { + "mean_centered_corr": -0.013281173072755337, + "mean_pairwise_disagreement": 0.09412748922670262, + "member_std": 0.06749773025512695 + }, + "test": { + "mean_centered_corr": -0.011601177044212818, + "mean_pairwise_disagreement": 0.059119563958273634, + "member_std": 0.06550896167755127 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 22, + 24, + 24 + ], + "head_selection_time": "0:00:20.435848" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/1/report.json new file mode 100644 index 000000000..62045c384 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/1/report.json @@ -0,0 +1,426 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + }, + "mask_stats": { + "keep_rate_actual": 0.9005681872367859, + "min_features_per_member": 89.0, + "mean_features_per_member": 99.0625, + "max_features_per_member": 106.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.81818199157715, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8186085273842617, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 428, + "metrics": { + "train": { + "0": { + "precision": 0.7888152627873106, + "recall": 0.9715249646586441, + "f1-score": 0.8706881866961929, + "support": 84179.0 + }, + "1": { + "precision": 0.5768008474576272, + "recall": 0.12983864557666322, + "f1-score": 0.21196392655550508, + "support": 25162.0 + }, + "accuracy": 0.7778326519786722, + "macro avg": { + "precision": 0.6828080551224689, + "recall": 0.5506818051176536, + "f1-score": 0.541326056625849, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.740025634756421, + "recall": 0.7778326519786722, + "f1-score": 0.7190998544725988, + "support": 109341.0 + }, + "cross-entropy": 0.497526106332481, + "roc-auc": 0.6852792162881653, + "score": 0.6852792162881653 + }, + "val": { + "0": { + "precision": 0.6946272468323348, + "recall": 0.9114576620698543, + "f1-score": 0.7884057971014493, + "support": 15518.0 + }, + "1": { + "precision": 0.6476019492177482, + "recall": 0.28880247054786684, + "f1-score": 0.39946211042556556, + "support": 8743.0 + }, + "accuracy": 0.6870697827789456, + "macro avg": { + "precision": 0.6711145980250415, + "recall": 0.6001300663088606, + "f1-score": 0.5939339537635074, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6776806173840709, + "recall": 0.6870697827789456, + "f1-score": 0.6482411438469564, + "support": 24261.0 + }, + "cross-entropy": 0.6172041286027814, + "roc-auc": 0.6512227217747169, + "score": 0.6512227217747169 + }, + "test": { + "0": { + "precision": 0.6737023929895517, + "recall": 0.9449828625457983, + "f1-score": 0.7866102612032072, + "support": 16922.0 + }, + "1": { + "precision": 0.6575947039352703, + "recall": 0.18755900555963495, + "f1-score": 0.29187071498530853, + "support": 9533.0 + }, + "accuracy": 0.6720468720468721, + "macro avg": { + "precision": 0.6656485484624111, + "recall": 0.5662709340527167, + "f1-score": 0.5392404880942578, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6678980233144632, + "recall": 0.6720468720468721, + "f1-score": 0.6083319737681201, + "support": 26455.0 + }, + "cross-entropy": 0.6427700166976323, + "roc-auc": 0.6075381341628896, + "score": 0.6075381341628896 + } + }, + "time": "0:01:23.222825", + "diversity": { + "train": { + "mean_centered_corr": -0.008371543139219284, + "mean_pairwise_disagreement": 0.02743249071932702, + "member_std": 0.027091743424534798 + }, + "val": { + "mean_centered_corr": 0.05517549440264702, + "mean_pairwise_disagreement": 0.07159946402762432, + "member_std": 0.054152194410562515 + }, + "test": { + "mean_centered_corr": 0.09162398427724838, + "mean_pairwise_disagreement": 0.04175463812560587, + "member_std": 0.052940964698791504 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 16 + ], + "head_selection_time": "0:00:11.118299" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/2/report.json new file mode 100644 index 000000000..704a44552 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/2/report.json @@ -0,0 +1,427 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + }, + "mask_stats": { + "keep_rate_actual": 0.8940340876579285, + "min_features_per_member": 89.0, + "mean_features_per_member": 98.34375, + "max_features_per_member": 104.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.60909080505371, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8074834695307933, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.787158853016306, + "recall": 0.9737583007638485, + "f1-score": 0.870571867051133, + "support": 84179.0 + }, + "1": { + "precision": 0.5757633954292298, + "recall": 0.11914792146888165, + "f1-score": 0.19743817708847838, + "support": 25162.0 + }, + "accuracy": 0.7770918502665971, + "macro avg": { + "precision": 0.6814611242227679, + "recall": 0.5464531111163651, + "f1-score": 0.5340050220698057, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7385116620832981, + "recall": 0.7770918502665971, + "f1-score": 0.7156675776552036, + "support": 109341.0 + }, + "cross-entropy": 0.49884451202024715, + "roc-auc": 0.6825760332150292, + "score": 0.6825760332150292 + }, + "val": { + "0": { + "precision": 0.6974369447564587, + "recall": 0.8802680757829617, + "f1-score": 0.7782588878760255, + "support": 15518.0 + }, + "1": { + "precision": 0.6025668449197861, + "recall": 0.3222006176369667, + "f1-score": 0.41988373826203607, + "support": 8743.0 + }, + "accuracy": 0.6791558468323647, + "macro avg": { + "precision": 0.6500018948381223, + "recall": 0.6012343467099642, + "f1-score": 0.5990713130690308, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6632483588419446, + "recall": 0.6791558468323647, + "f1-score": 0.6491102982434831, + "support": 24261.0 + }, + "cross-entropy": 0.6255965554487835, + "roc-auc": 0.6436592869751769, + "score": 0.6436592869751769 + }, + "test": { + "0": { + "precision": 0.6723253228427264, + "recall": 0.9291454910767049, + "f1-score": 0.7801428996725216, + "support": 16922.0 + }, + "1": { + "precision": 0.6093189964157706, + "recall": 0.19616070491975243, + "f1-score": 0.29677828916045074, + "support": 9533.0 + }, + "accuracy": 0.665016065016065, + "macro avg": { + "precision": 0.6408221596292485, + "recall": 0.5626530979982286, + "f1-score": 0.5384605944164862, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6496211342270329, + "recall": 0.665016065016065, + "f1-score": 0.6059635448431293, + "support": 26455.0 + }, + "cross-entropy": 0.6535820200884032, + "roc-auc": 0.6015328994897303, + "score": 0.6015328994897303 + } + }, + "time": "0:01:18.718791", + "diversity": { + "train": { + "mean_centered_corr": -0.007062356453388929, + "mean_pairwise_disagreement": 0.025052193183149135, + "member_std": 0.028095407411456108 + }, + "val": { + "mean_centered_corr": -0.003141294466331601, + "mean_pairwise_disagreement": 0.09142868017301098, + "member_std": 0.06040506064891815 + }, + "test": { + "mean_centered_corr": 0.00333536509424448, + "mean_pairwise_disagreement": 0.05701769895318282, + "member_std": 0.059019479900598526 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 23, + 9 + ], + "head_selection_time": "0:00:15.951706" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..223176d4b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-best-head-evaluation/0/report.json @@ -0,0 +1,229 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.001-evaluation/_esam_diagnostics/esam_rho0.001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7841790931223371, + "recall": 0.9795554710794854, + "f1-score": 0.8710459459887709, + "support": 84179.0 + }, + "1": { + "precision": 0.5891620911912151, + "recall": 0.0980844130037358, + "f1-score": 0.16817144219958435, + "support": 25162.0 + }, + "accuracy": 0.7767077308603361, + "macro avg": { + "precision": 0.6866705921567762, + "recall": 0.5388199420416105, + "f1-score": 0.5196086940941776, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7393009796736684, + "recall": 0.7767077308603361, + "f1-score": 0.709297578365066, + "support": 109341.0 + }, + "cross-entropy": 0.4992703210863206, + "roc-auc": 0.6803857932728635, + "score": 0.6803857932728635 + }, + "val": { + "0": { + "precision": 0.688133775351014, + "recall": 0.9095888645444, + "f1-score": 0.7835137385512073, + "support": 15518.0 + }, + "1": { + "precision": 0.6257668711656442, + "recall": 0.26832894887338443, + "f1-score": 0.3756003842459174, + "support": 8743.0 + }, + "accuracy": 0.6784963521701496, + "macro avg": { + "precision": 0.6569503232583291, + "recall": 0.5889589067088923, + "f1-score": 0.5795570613985623, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6656584510324497, + "recall": 0.6784963521701496, + "f1-score": 0.63651293657721, + "support": 24261.0 + }, + "cross-entropy": 0.6310932933009203, + "roc-auc": 0.6410241296714208, + "score": 0.6410241296714208 + }, + "test": { + "0": { + "precision": 0.6670702684647822, + "recall": 0.9441555371705472, + "f1-score": 0.781787488072811, + "support": 16922.0 + }, + "1": { + "precision": 0.6226038338658147, + "recall": 0.16353718661491662, + "f1-score": 0.2590346431835175, + "support": 9533.0 + }, + "accuracy": 0.6628614628614629, + "macro avg": { + "precision": 0.6448370511652985, + "recall": 0.553846361892732, + "f1-score": 0.5204110656281642, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6510468883463941, + "recall": 0.6628614628614629, + "f1-score": 0.5934146712015339, + "support": 26455.0 + }, + "cross-entropy": 0.6635058051130894, + "roc-auc": 0.6024966050474919, + "score": 0.6024966050474919 + } + }, + "time": "0:01:33.915789", + "diversity": { + "train": { + "mean_centered_corr": -0.013111491687595844, + "mean_pairwise_disagreement": 0.02585577938919114, + "member_std": 0.02714134193956852 + }, + "val": { + "mean_centered_corr": -0.004636597353965044, + "mean_pairwise_disagreement": 0.08690653790565238, + "member_std": 0.060862503945827484 + }, + "test": { + "mean_centered_corr": 0.0027443149592727423, + "mean_pairwise_disagreement": 0.05465580626870949, + "member_std": 0.05927826836705208 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0 + ], + "head_selection_time": "0:00:04.756597" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-best-head-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-best-head-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-best-head-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-best-head-evaluation/1/report.json new file mode 100644 index 000000000..6ae6a0c85 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-best-head-evaluation/1/report.json @@ -0,0 +1,229 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__1.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 428, + "metrics": { + "train": { + "0": { + "precision": 0.7921965289755508, + "recall": 0.9657515532377434, + "f1-score": 0.870406852248394, + "support": 84179.0 + }, + "1": { + "precision": 0.5709821428571429, + "recall": 0.15249185279389554, + "f1-score": 0.2407000815507183, + "support": 25162.0 + }, + "accuracy": 0.7786008907911945, + "macro avg": { + "precision": 0.6815893359163469, + "recall": 0.5591217030158195, + "f1-score": 0.5555534668995561, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7412897658810906, + "recall": 0.7786008907911945, + "f1-score": 0.7254961438746375, + "support": 109341.0 + }, + "cross-entropy": 0.49324286572021286, + "roc-auc": 0.6956759811999328, + "score": 0.6956759811999328 + }, + "val": { + "0": { + "precision": 0.6824444652732214, + "recall": 0.9383941229539889, + "f1-score": 0.7902105491643152, + "support": 15518.0 + }, + "1": { + "precision": 0.6729387615463565, + "recall": 0.22497998398718974, + "f1-score": 0.3372192696725527, + "support": 8743.0 + }, + "accuracy": 0.6812992044845637, + "macro avg": { + "precision": 0.677691613409789, + "recall": 0.5816870534705894, + "f1-score": 0.5637149094184339, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6790188699686593, + "recall": 0.6812992044845637, + "f1-score": 0.6269648974353478, + "support": 24261.0 + }, + "cross-entropy": 0.6631863406542986, + "roc-auc": 0.6315182980623079, + "score": 0.6315182980623079 + }, + "test": { + "0": { + "precision": 0.6619854721549637, + "recall": 0.9693889611157074, + "f1-score": 0.7867248573209918, + "support": 16922.0 + }, + "1": { + "precision": 0.6907462686567164, + "recall": 0.12136787999580405, + "f1-score": 0.20645967166309778, + "support": 9533.0 + }, + "accuracy": 0.6638064638064638, + "macro avg": { + "precision": 0.6763658704058401, + "recall": 0.5453784205557557, + "f1-score": 0.4965922644920448, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.672349360760188, + "recall": 0.6638064638064638, + "f1-score": 0.5776275972613923, + "support": 26455.0 + }, + "cross-entropy": 0.7198854018171156, + "roc-auc": 0.5962485571769538, + "score": 0.5962485571769538 + } + }, + "time": "0:01:39.396326", + "diversity": { + "train": { + "mean_centered_corr": -0.00980177242308855, + "mean_pairwise_disagreement": 0.02483125814446725, + "member_std": 0.02443780191242695 + }, + "val": { + "mean_centered_corr": 0.02159964293241501, + "mean_pairwise_disagreement": 0.06877450667538902, + "member_std": 0.04672974348068237 + }, + "test": { + "mean_centered_corr": 0.04072229564189911, + "mean_pairwise_disagreement": 0.03965246828150054, + "member_std": 0.04293917864561081 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 15 + ], + "head_selection_time": "0:00:04.804613" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-best-head-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-best-head-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-best-head-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-best-head-evaluation/2/report.json new file mode 100644 index 000000000..b7ffa6124 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-best-head-evaluation/2/report.json @@ -0,0 +1,229 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__2.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7891122957469, + "recall": 0.9706815238955084, + "f1-score": 0.8705300782519296, + "support": 84179.0 + }, + "1": { + "precision": 0.5739685827723114, + "recall": 0.13214370876718862, + "f1-score": 0.2148279760943305, + "support": 25162.0 + }, + "accuracy": 0.7777137578767342, + "macro avg": { + "precision": 0.6815404392596057, + "recall": 0.5514126163313485, + "f1-score": 0.5426790271731301, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7396025408894668, + "recall": 0.7777137578767342, + "f1-score": 0.7196372174358633, + "support": 109341.0 + }, + "cross-entropy": 0.4988365072053171, + "roc-auc": 0.6810069006558737, + "score": 0.6810069006558737 + }, + "val": { + "0": { + "precision": 0.6966991903674487, + "recall": 0.8650599304034025, + "f1-score": 0.7718047490369689, + "support": 15518.0 + }, + "1": { + "precision": 0.5806128580012017, + "recall": 0.3315795493537687, + "f1-score": 0.4221025043680839, + "support": 8743.0 + }, + "accuracy": 0.6728082107085446, + "macro avg": { + "precision": 0.6386560241843251, + "recall": 0.5983197398785856, + "f1-score": 0.5969536267025264, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6548648552667481, + "recall": 0.6728082107085446, + "f1-score": 0.6457816368346664, + "support": 24261.0 + }, + "cross-entropy": 0.6306857322286159, + "roc-auc": 0.6341914987995403, + "score": 0.6341914987995403 + }, + "test": { + "0": { + "precision": 0.6714620131942967, + "recall": 0.9322775085687272, + "f1-score": 0.7806616027909048, + "support": 16922.0 + }, + "1": { + "precision": 0.6128378378378379, + "recall": 0.1902863736494283, + "f1-score": 0.29040262547026335, + "support": 9533.0 + }, + "accuracy": 0.6649026649026649, + "macro avg": { + "precision": 0.6421499255160672, + "recall": 0.5612819411090777, + "f1-score": 0.5355321141305841, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6503369229023624, + "recall": 0.6649026649026649, + "f1-score": 0.6039978783230282, + "support": 26455.0 + }, + "cross-entropy": 0.6517993050987927, + "roc-auc": 0.599267480873393, + "score": 0.599267480873393 + } + }, + "time": "0:01:33.653112", + "diversity": { + "train": { + "mean_centered_corr": -0.015336746349930763, + "mean_pairwise_disagreement": 0.023258879958555228, + "member_std": 0.02414415217936039 + }, + "val": { + "mean_centered_corr": -0.00032689498038962483, + "mean_pairwise_disagreement": 0.07033457387470399, + "member_std": 0.050614725798368454 + }, + "test": { + "mean_centered_corr": 0.007613664958626032, + "mean_pairwise_disagreement": 0.041485389066034224, + "member_std": 0.04775906354188919 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31 + ], + "head_selection_time": "0:00:04.781830" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-evaluation/0.toml new file mode 100644 index 000000000..9b75584bc --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-evaluation/0.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.001 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-evaluation/0/report.json new file mode 100644 index 000000000..517e309dc --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-evaluation/0/report.json @@ -0,0 +1,225 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.001-evaluation/_esam_diagnostics/esam_rho0.001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7862810438170169, + "recall": 0.9782487318689934, + "f1-score": 0.8718225610078874, + "support": 84179.0 + }, + "1": { + "precision": 0.602819956616052, + "recall": 0.11044432080120817, + "f1-score": 0.18668547628644364, + "support": 25162.0 + }, + "accuracy": 0.7785460165903001, + "macro avg": { + "precision": 0.6945505002165344, + "recall": 0.5443465263351008, + "f1-score": 0.5292540186471655, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7440622249279388, + "recall": 0.7785460165903001, + "f1-score": 0.7141560011103103, + "support": 109341.0 + }, + "cross-entropy": 0.4922902286245573, + "roc-auc": 0.6989728271677539, + "score": 0.6989728271677539 + }, + "val": { + "0": { + "precision": 0.675189954047045, + "recall": 0.9563088026807578, + "f1-score": 0.791529989065792, + "support": 15518.0 + }, + "1": { + "precision": 0.7028921998247152, + "recall": 0.18346105455793205, + "f1-score": 0.2909750566893424, + "support": 8743.0 + }, + "accuracy": 0.677795639091546, + "macro avg": { + "precision": 0.6890410769358801, + "recall": 0.5698849286193449, + "f1-score": 0.5412525228775672, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6851730847850266, + "recall": 0.677795639091546, + "f1-score": 0.6111436993923531, + "support": 24261.0 + }, + "cross-entropy": 0.6872600691770208, + "roc-auc": 0.6227874903903754, + "score": 0.6227874903903754 + }, + "test": { + "0": { + "precision": 0.656414727914074, + "recall": 0.9787259189221132, + "f1-score": 0.7858040946077385, + "support": 16922.0 + }, + "1": { + "precision": 0.7058823529411765, + "recall": 0.09063253959928669, + "f1-score": 0.16063958352700566, + "support": 9533.0 + }, + "accuracy": 0.6587034587034587, + "macro avg": { + "precision": 0.6811485404276252, + "recall": 0.5346792292607, + "f1-score": 0.4732218390673721, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6742402758023133, + "recall": 0.6587034587034587, + "f1-score": 0.5605274631908937, + "support": 26455.0 + }, + "cross-entropy": 0.7564828184565525, + "roc-auc": 0.5911993816464689, + "score": 0.5911993816464689 + } + }, + "time": "0:01:33.915789", + "diversity": { + "train": { + "mean_centered_corr": -0.013111491687595844, + "mean_pairwise_disagreement": 0.02585577938919114, + "member_std": 0.02714134193956852 + }, + "val": { + "mean_centered_corr": -0.004636597353965044, + "mean_pairwise_disagreement": 0.08690653790565238, + "member_std": 0.060862503945827484 + }, + "test": { + "mean_centered_corr": 0.0027443149592727423, + "mean_pairwise_disagreement": 0.05465580626870949, + "member_std": 0.05927826836705208 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-evaluation/1.toml new file mode 100644 index 000000000..b80a19935 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-evaluation/1.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 1 +head_selection = true +use_esam = true +esam_rho = 0.001 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-evaluation/1/report.json new file mode 100644 index 000000000..d79e09c49 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-evaluation/1/report.json @@ -0,0 +1,225 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__1.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 428, + "metrics": { + "train": { + "0": { + "precision": 0.7880230326295585, + "recall": 0.9754451822901199, + "f1-score": 0.8717744546897478, + "support": 84179.0 + }, + "1": { + "precision": 0.5979381443298969, + "recall": 0.12216834909784596, + "f1-score": 0.2028842028842029, + "support": 25162.0 + }, + "accuracy": 0.7790856128990955, + "macro avg": { + "precision": 0.6929805884797278, + "recall": 0.5488067656939829, + "f1-score": 0.5373293287869754, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.744279917426697, + "recall": 0.7790856128990955, + "f1-score": 0.7178466827109738, + "support": 109341.0 + }, + "cross-entropy": 0.49036011907714316, + "roc-auc": 0.7032037382378304, + "score": 0.7032037382378304 + }, + "val": { + "0": { + "precision": 0.6734879077998094, + "recall": 0.9565021265627014, + "f1-score": 0.7904252203317624, + "support": 15518.0 + }, + "1": { + "precision": 0.6962196219621962, + "recall": 0.17694155324259409, + "f1-score": 0.28217054263565894, + "support": 8743.0 + }, + "accuracy": 0.6755698446065702, + "macro avg": { + "precision": 0.6848537648810028, + "recall": 0.5667218399026477, + "f1-score": 0.5362978814837107, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6816797950642152, + "recall": 0.6755698446065702, + "f1-score": 0.6072641533066179, + "support": 24261.0 + }, + "cross-entropy": 0.7136592088387013, + "roc-auc": 0.6204141557865444, + "score": 0.6204141557865444 + }, + "test": { + "0": { + "precision": 0.6556994100645366, + "recall": 0.9786668242524524, + "f1-score": 0.7852722918988122, + "support": 16922.0 + }, + "1": { + "precision": 0.6986644407345576, + "recall": 0.08780027273680897, + "f1-score": 0.15599664523343584, + "support": 9533.0 + }, + "accuracy": 0.6576450576450577, + "macro avg": { + "precision": 0.677181925399547, + "recall": 0.5332335484946307, + "f1-score": 0.470634468566124, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6711817626397515, + "recall": 0.6576450576450577, + "f1-score": 0.5585142219815552, + "support": 26455.0 + }, + "cross-entropy": 0.7976474685558286, + "roc-auc": 0.5897760078319128, + "score": 0.5897760078319128 + } + }, + "time": "0:01:39.396326", + "diversity": { + "train": { + "mean_centered_corr": -0.00980177242308855, + "mean_pairwise_disagreement": 0.02483125814446725, + "member_std": 0.02443780191242695 + }, + "val": { + "mean_centered_corr": 0.02159964293241501, + "mean_pairwise_disagreement": 0.06877450667538902, + "member_std": 0.04672974348068237 + }, + "test": { + "mean_centered_corr": 0.04072229564189911, + "mean_pairwise_disagreement": 0.03965246828150054, + "member_std": 0.04293917864561081 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-evaluation/2.toml new file mode 100644 index 000000000..029b6e363 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-evaluation/2.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 2 +head_selection = true +use_esam = true +esam_rho = 0.001 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-evaluation/2/report.json new file mode 100644 index 000000000..ab12d4221 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-evaluation/2/report.json @@ -0,0 +1,225 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__2.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7859541664281489, + "recall": 0.9786169947374048, + "f1-score": 0.8717677374294286, + "support": 84179.0 + }, + "1": { + "precision": 0.6023856858846919, + "recall": 0.10837771242349575, + "f1-score": 0.1837044023038836, + "support": 25162.0 + }, + "accuracy": 0.7783539568871695, + "macro avg": { + "precision": 0.6941699261564204, + "recall": 0.5434973535804503, + "f1-score": 0.5277360698666561, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7437106337420158, + "recall": 0.7783539568871695, + "f1-score": 0.7134277767703074, + "support": 109341.0 + }, + "cross-entropy": 0.4919189215910807, + "roc-auc": 0.6999636786911775, + "score": 0.6999636786911775 + }, + "val": { + "0": { + "precision": 0.6763377583139455, + "recall": 0.9554066245650212, + "f1-score": 0.7920083335559176, + "support": 15518.0 + }, + "1": { + "precision": 0.7042735042735043, + "recall": 0.18849365206450874, + "f1-score": 0.29739240277903095, + "support": 8743.0 + }, + "accuracy": 0.6790321915831994, + "macro avg": { + "precision": 0.6903056312937249, + "recall": 0.5719501383147649, + "f1-score": 0.5447003681674742, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6864050361229568, + "recall": 0.6790321915831994, + "f1-score": 0.6137622974163388, + "support": 24261.0 + }, + "cross-entropy": 0.7025996858462662, + "roc-auc": 0.6211550537725488, + "score": 0.6211550537725488 + }, + "test": { + "0": { + "precision": 0.6576476205634251, + "recall": 0.9767167001536461, + "f1-score": 0.7860370000475579, + "support": 16922.0 + }, + "1": { + "precision": 0.7021919879062736, + "recall": 0.09745095982377006, + "f1-score": 0.17114959469417834, + "support": 9533.0 + }, + "accuracy": 0.6598752598752599, + "macro avg": { + "precision": 0.6799198042348493, + "recall": 0.5370838299887081, + "f1-score": 0.47859329737086814, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6736990835715285, + "recall": 0.6598752598752599, + "f1-score": 0.5644637006624221, + "support": 26455.0 + }, + "cross-entropy": 0.7802766696915584, + "roc-auc": 0.5892400551940371, + "score": 0.5892400551940371 + } + }, + "time": "0:01:33.653112", + "diversity": { + "train": { + "mean_centered_corr": -0.015336746349930763, + "mean_pairwise_disagreement": 0.023258879958555228, + "member_std": 0.02414415217936039 + }, + "val": { + "mean_centered_corr": -0.00032689498038962483, + "mean_pairwise_disagreement": 0.07033457387470399, + "member_std": 0.050614725798368454 + }, + "test": { + "mean_centered_corr": 0.007613664958626032, + "mean_pairwise_disagreement": 0.041485389066034224, + "member_std": 0.04775906354188919 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..a92177d15 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,230 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.001-evaluation/_esam_diagnostics/esam_rho0.001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.782973008757495, + "recall": 0.9834994476056974, + "f1-score": 0.871854546986315, + "support": 84179.0 + }, + "1": { + "precision": 0.6144879267277269, + "recall": 0.08798982592798664, + "f1-score": 0.1539370763080132, + "support": 25162.0 + }, + "accuracy": 0.7774210954719638, + "macro avg": { + "precision": 0.6987304677426109, + "recall": 0.535744636766842, + "f1-score": 0.5128958116471641, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7442005296871277, + "recall": 0.7774210954719638, + "f1-score": 0.7066444300383502, + "support": 109341.0 + }, + "cross-entropy": 0.4979289521091686, + "roc-auc": 0.6847935169951291, + "score": 0.6847935169951291 + }, + "val": { + "0": { + "precision": 0.6776083871263104, + "recall": 0.9537955922154917, + "f1-score": 0.7923235459436311, + "support": 15518.0 + }, + "1": { + "precision": 0.7034739454094293, + "recall": 0.1945556445156125, + "f1-score": 0.3048113968282412, + "support": 8743.0 + }, + "accuracy": 0.6801863072420757, + "macro avg": { + "precision": 0.6905411662678699, + "recall": 0.5741756183655521, + "f1-score": 0.5485674713859361, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6869296259898902, + "recall": 0.6801863072420757, + "f1-score": 0.6166375181741305, + "support": 24261.0 + }, + "cross-entropy": 0.639843807695552, + "roc-auc": 0.641845068859757, + "score": 0.641845068859757 + }, + "test": { + "0": { + "precision": 0.6588995730758489, + "recall": 0.975889374778395, + "f1-score": 0.7866619030606169, + "support": 16922.0 + }, + "1": { + "precision": 0.7068965517241379, + "recall": 0.10322039232140984, + "f1-score": 0.18013729977116705, + "support": 9533.0 + }, + "accuracy": 0.6614250614250614, + "macro avg": { + "precision": 0.6828980623999934, + "recall": 0.5395548835499024, + "f1-score": 0.483399601415892, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6761951768352191, + "recall": 0.6614250614250614, + "f1-score": 0.5681021206694499, + "support": 26455.0 + }, + "cross-entropy": 0.6818687369091303, + "roc-auc": 0.6041622806453656, + "score": 0.6041622806453656 + } + }, + "time": "0:01:33.915789", + "diversity": { + "train": { + "mean_centered_corr": -0.013111491687595844, + "mean_pairwise_disagreement": 0.02585577938919114, + "member_std": 0.02714134193956852 + }, + "val": { + "mean_centered_corr": -0.004636597353965044, + "mean_pairwise_disagreement": 0.08690653790565238, + "member_std": 0.060862503945827484 + }, + "test": { + "mean_centered_corr": 0.0027443149592727423, + "mean_pairwise_disagreement": 0.05465580626870949, + "member_std": 0.05927826836705208 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0, + 25 + ], + "head_selection_time": "0:00:15.827629" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/1/report.json new file mode 100644 index 000000000..4e16edaf9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/1/report.json @@ -0,0 +1,233 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__1.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 428, + "metrics": { + "train": { + "0": { + "precision": 0.7931243640917274, + "recall": 0.9630667981325509, + "f1-score": 0.869873118914134, + "support": 84179.0 + }, + "1": { + "precision": 0.5636491228070175, + "recall": 0.15960575470948254, + "f1-score": 0.2487688543376591, + "support": 25162.0 + }, + "accuracy": 0.778171042884188, + "macro avg": { + "precision": 0.6783867434493724, + "recall": 0.5613362764210167, + "f1-score": 0.5593209866258966, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7403165790778179, + "recall": 0.778171042884188, + "f1-score": 0.7269420545807801, + "support": 109341.0 + }, + "cross-entropy": 0.4937839932106913, + "roc-auc": 0.6959329827185088, + "score": 0.6959329827185088 + }, + "val": { + "0": { + "precision": 0.6899277108433735, + "recall": 0.9225415646346179, + "f1-score": 0.7894562699900739, + "support": 15518.0 + }, + "1": { + "precision": 0.6576473939048704, + "recall": 0.26409699187921765, + "f1-score": 0.3768565366410968, + "support": 8743.0 + }, + "accuracy": 0.6852561724578542, + "macro avg": { + "precision": 0.6737875523741219, + "recall": 0.5933192782569178, + "f1-score": 0.5831564033155854, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6782947686318681, + "recall": 0.6852561724578542, + "f1-score": 0.6407666253476393, + "support": 24261.0 + }, + "cross-entropy": 0.6548096037426314, + "roc-auc": 0.6326258289049813, + "score": 0.6326258289049813 + }, + "test": { + "0": { + "precision": 0.6649503493931592, + "recall": 0.9615884647204822, + "f1-score": 0.7862198922523131, + "support": 16922.0 + }, + "1": { + "precision": 0.6723790322580645, + "recall": 0.1399349627609357, + "f1-score": 0.23165754970912564, + "support": 9533.0 + }, + "accuracy": 0.6655074655074655, + "macro avg": { + "precision": 0.6686646908256119, + "recall": 0.5507617137407089, + "f1-score": 0.5089387209807194, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6676272586258615, + "recall": 0.6655074655074655, + "f1-score": 0.5863845941436681, + "support": 26455.0 + }, + "cross-entropy": 0.7047260741581922, + "roc-auc": 0.5972452690882881, + "score": 0.5972452690882881 + } + }, + "time": "0:01:39.396326", + "diversity": { + "train": { + "mean_centered_corr": -0.00980177242308855, + "mean_pairwise_disagreement": 0.02483125814446725, + "member_std": 0.02443780191242695 + }, + "val": { + "mean_centered_corr": 0.02159964293241501, + "mean_pairwise_disagreement": 0.06877450667538902, + "member_std": 0.04672974348068237 + }, + "test": { + "mean_centered_corr": 0.04072229564189911, + "mean_pairwise_disagreement": 0.03965246828150054, + "member_std": 0.04293917864561081 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 15, + 10, + 9, + 20, + 9 + ], + "head_selection_time": "0:00:29.881915" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/2/report.json new file mode 100644 index 000000000..d7841fe53 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/2/report.json @@ -0,0 +1,230 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__2.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7868832039858197, + "recall": 0.9756233740006415, + "f1-score": 0.8711475531559435, + "support": 84179.0 + }, + "1": { + "precision": 0.5872057936028968, + "recall": 0.11600826643351085, + "f1-score": 0.1937410812066505, + "support": 25162.0 + }, + "accuracy": 0.777805214878225, + "macro avg": { + "precision": 0.6870444987943582, + "recall": 0.5458158202170762, + "f1-score": 0.532444317181297, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7409326182032212, + "recall": 0.777805214878225, + "f1-score": 0.7152599936202879, + "support": 109341.0 + }, + "cross-entropy": 0.49847322238722597, + "roc-auc": 0.682215318814317, + "score": 0.682215318814317 + }, + "val": { + "0": { + "precision": 0.6895468820098699, + "recall": 0.8914164196417064, + "f1-score": 0.7775935242699346, + "support": 15518.0 + }, + "1": { + "precision": 0.5988095238095238, + "recall": 0.28765869838728125, + "f1-score": 0.38862705709650003, + "support": 8743.0 + }, + "accuracy": 0.6738386711182557, + "macro avg": { + "precision": 0.6441782029096969, + "recall": 0.5895375590144938, + "f1-score": 0.5831102906832173, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6568476230038262, + "recall": 0.6738386711182557, + "f1-score": 0.63742066154798, + "support": 24261.0 + }, + "cross-entropy": 0.6333810186447947, + "roc-auc": 0.6354981541987959, + "score": 0.6354981541987959 + }, + "test": { + "0": { + "precision": 0.6670303844216887, + "recall": 0.9392506795887011, + "f1-score": 0.7800736196319018, + "support": 16922.0 + }, + "1": { + "precision": 0.6086791016368481, + "recall": 0.167733137522291, + "f1-score": 0.26299342105263157, + "support": 9533.0 + }, + "accuracy": 0.6612360612360613, + "macro avg": { + "precision": 0.6378547430292685, + "recall": 0.553491908555496, + "f1-score": 0.5215335203422666, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6460036303567527, + "recall": 0.6612360612360613, + "f1-score": 0.5937449281536865, + "support": 26455.0 + }, + "cross-entropy": 0.6626198047400376, + "roc-auc": 0.5998512677731419, + "score": 0.5998512677731419 + } + }, + "time": "0:01:33.653112", + "diversity": { + "train": { + "mean_centered_corr": -0.015336746349930763, + "mean_pairwise_disagreement": 0.023258879958555228, + "member_std": 0.02414415217936039 + }, + "val": { + "mean_centered_corr": -0.00032689498038962483, + "mean_pairwise_disagreement": 0.07033457387470399, + "member_std": 0.050614725798368454 + }, + "test": { + "mean_centered_corr": 0.007613664958626032, + "mean_pairwise_disagreement": 0.041485389066034224, + "member_std": 0.04775906354188919 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31, + 20 + ], + "head_selection_time": "0:00:15.768462" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..85c7d6ec0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/0/report.json @@ -0,0 +1,255 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.6894886493682861, + "min_features_per_member": 67.0, + "mean_features_per_member": 75.84375, + "max_features_per_member": 84.0, + "min_members_per_feature": 14.0, + "mean_members_per_feature": 22.063636779785156, + "max_members_per_feature": 31.0, + "mean_pairwise_jaccard": 0.5278457608823273, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 428, + "metrics": { + "train": { + "0": { + "precision": 0.7894966160281175, + "recall": 0.9686501383955618, + "f1-score": 0.8699455883921904, + "support": 84179.0 + }, + "1": { + "precision": 0.5645214521452145, + "recall": 0.13595898577219617, + "f1-score": 0.21914034975337904, + "support": 25162.0 + }, + "accuracy": 0.7770278303655537, + "macro avg": { + "precision": 0.677009034086666, + "recall": 0.552304562083879, + "f1-score": 0.5445429690727848, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7377244073084094, + "recall": 0.7770278303655537, + "f1-score": 0.7201796139212255, + "support": 109341.0 + }, + "cross-entropy": 0.5002029863194409, + "roc-auc": 0.6821879786641952, + "score": 0.6821879786641952 + }, + "val": { + "0": { + "precision": 0.6961402811584124, + "recall": 0.9030802938523006, + "f1-score": 0.7862212123762238, + "support": 15518.0 + }, + "1": { + "precision": 0.6358353510895883, + "recall": 0.30035456936978155, + "f1-score": 0.4079857065175173, + "support": 8743.0 + }, + "accuracy": 0.6858744487036808, + "macro avg": { + "precision": 0.6659878161240004, + "recall": 0.6017174316110411, + "f1-score": 0.5971034594468705, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6744080358432264, + "recall": 0.6858744487036808, + "f1-score": 0.6499154942391862, + "support": 24261.0 + }, + "cross-entropy": 0.6154470541736884, + "roc-auc": 0.6547032702847417, + "score": 0.6547032702847417 + }, + "test": { + "0": { + "precision": 0.6733296707940667, + "recall": 0.9415553717054722, + "f1-score": 0.7851669335961562, + "support": 16922.0 + }, + "1": { + "precision": 0.6457736389684814, + "recall": 0.18913248714990036, + "f1-score": 0.2925760649087221, + "support": 9533.0 + }, + "accuracy": 0.6704214704214704, + "macro avg": { + "precision": 0.6595516548812741, + "recall": 0.5653439294276863, + "f1-score": 0.5388714992524392, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6633999164416454, + "recall": 0.6704214704214704, + "f1-score": 0.6076629172968816, + "support": 26455.0 + }, + "cross-entropy": 0.638677137478482, + "roc-auc": 0.6118564029158264, + "score": 0.6118564029158264 + } + }, + "time": "0:01:10.628940", + "diversity": { + "train": { + "mean_centered_corr": -0.015634549781680107, + "mean_pairwise_disagreement": 0.03130809179096706, + "member_std": 0.0355970673263073 + }, + "val": { + "mean_centered_corr": -0.0187542624771595, + "mean_pairwise_disagreement": 0.12205770312369114, + "member_std": 0.07598911970853806 + }, + "test": { + "mean_centered_corr": -0.018549803644418716, + "mean_pairwise_disagreement": 0.08149855811146134, + "member_std": 0.0770004391670227 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 16 + ], + "head_selection_time": "0:00:04.834038" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/1/report.json new file mode 100644 index 000000000..5f56a5253 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/1/report.json @@ -0,0 +1,255 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + }, + "mask_stats": { + "keep_rate_actual": 0.703125, + "min_features_per_member": 67.0, + "mean_features_per_member": 77.34375, + "max_features_per_member": 88.0, + "min_members_per_feature": 14.0, + "mean_members_per_feature": 22.5, + "max_members_per_feature": 29.0, + "mean_pairwise_jaccard": 0.5422206532495487, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 428, + "metrics": { + "train": { + "0": { + "precision": 0.7894095930260653, + "recall": 0.9703251404744652, + "f1-score": 0.8705675459632294, + "support": 84179.0 + }, + "1": { + "precision": 0.5744463373083475, + "recall": 0.13401160480089022, + "f1-score": 0.21732405259087392, + "support": 25162.0 + }, + "accuracy": 0.7778692347792685, + "macro avg": { + "precision": 0.6819279651672063, + "recall": 0.5521683726376777, + "f1-score": 0.5439457992770517, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7399413657337485, + "recall": 0.7778692347792685, + "f1-score": 0.7202404702987009, + "support": 109341.0 + }, + "cross-entropy": 0.4976395217774722, + "roc-auc": 0.6850579661840904, + "score": 0.6850579661840904 + }, + "val": { + "0": { + "precision": 0.7023705004389815, + "recall": 0.8764015981440907, + "f1-score": 0.779794157277601, + "support": 15518.0 + }, + "1": { + "precision": 0.6084115965700286, + "recall": 0.3408441038545122, + "f1-score": 0.43691811450773405, + "support": 8743.0 + }, + "accuracy": 0.6834013437203743, + "macro avg": { + "precision": 0.6553910485045051, + "recall": 0.6086228509993015, + "f1-score": 0.6083561358926676, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6685102845976619, + "recall": 0.6834013437203743, + "f1-score": 0.6562310213006443, + "support": 24261.0 + }, + "cross-entropy": 0.617130934303606, + "roc-auc": 0.6525683493050402, + "score": 0.6525683493050402 + }, + "test": { + "0": { + "precision": 0.678315295344895, + "recall": 0.9222314147263917, + "f1-score": 0.7816874953041649, + "support": 16922.0 + }, + "1": { + "precision": 0.6183294663573086, + "recall": 0.22364418336305467, + "f1-score": 0.3284800862799476, + "support": 9533.0 + }, + "accuracy": 0.6704970704970705, + "macro avg": { + "precision": 0.6483223808511018, + "recall": 0.5729377990447232, + "f1-score": 0.5550837907920563, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6566995362166145, + "recall": 0.6704970704970705, + "f1-score": 0.6183752204892768, + "support": 26455.0 + }, + "cross-entropy": 0.6387111341102191, + "roc-auc": 0.6078370975247275, + "score": 0.6078370975247275 + } + }, + "time": "0:01:12.475731", + "diversity": { + "train": { + "mean_centered_corr": -0.013842575252056122, + "mean_pairwise_disagreement": 0.028755187603386977, + "member_std": 0.03266480937600136 + }, + "val": { + "mean_centered_corr": -0.0060155028477311134, + "mean_pairwise_disagreement": 0.08212387197825795, + "member_std": 0.06719459593296051 + }, + "test": { + "mean_centered_corr": 0.0020387060940265656, + "mean_pairwise_disagreement": 0.052944440041214236, + "member_std": 0.0697987824678421 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 16 + ], + "head_selection_time": "0:00:04.894692" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/2/report.json new file mode 100644 index 000000000..e19938a28 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/2/report.json @@ -0,0 +1,255 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + }, + "mask_stats": { + "keep_rate_actual": 0.7090908885002136, + "min_features_per_member": 65.0, + "mean_features_per_member": 78.0, + "max_features_per_member": 92.0, + "min_members_per_feature": 17.0, + "mean_members_per_feature": 22.690908432006836, + "max_members_per_feature": 28.0, + "mean_pairwise_jaccard": 0.5463052047602153, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7825240884186662, + "recall": 0.9840696610793666, + "f1-score": 0.8717999989475844, + "support": 84179.0 + }, + "1": { + "precision": 0.6147658718758977, + "recall": 0.08504888323662665, + "f1-score": 0.14942568864993191, + "support": 25162.0 + }, + "accuracy": 0.7771833072680879, + "macro avg": { + "precision": 0.6986449801472819, + "recall": 0.5345592721579966, + "f1-score": 0.5106128437987582, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7439188786195136, + "recall": 0.7771833072680879, + "f1-score": 0.7055642557615013, + "support": 109341.0 + }, + "cross-entropy": 0.49927567921826405, + "roc-auc": 0.6841046410049181, + "score": 0.6841046410049181 + }, + "val": { + "0": { + "precision": 0.6951219512195121, + "recall": 0.9256347467457147, + "f1-score": 0.7939859598695483, + "support": 15518.0 + }, + "1": { + "precision": 0.6791770920211287, + "recall": 0.27942353883106485, + "f1-score": 0.3959481361426256, + "support": 8743.0 + }, + "accuracy": 0.6927579242405507, + "macro avg": { + "precision": 0.6871495216203205, + "recall": 0.6025291427883898, + "f1-score": 0.5949670480060869, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.689375860622609, + "recall": 0.6927579242405507, + "f1-score": 0.6505440286694953, + "support": 24261.0 + }, + "cross-entropy": 0.6149192758236253, + "roc-auc": 0.6546592087434608, + "score": 0.6546592087434608 + }, + "test": { + "0": { + "precision": 0.6737775546910947, + "recall": 0.9518969388961116, + "f1-score": 0.7890470009062186, + "support": 16922.0 + }, + "1": { + "precision": 0.6805337519623234, + "recall": 0.18189447183467952, + "f1-score": 0.2870623292773777, + "support": 9533.0 + }, + "accuracy": 0.6744282744282745, + "macro avg": { + "precision": 0.677155653326709, + "recall": 0.5668957053653956, + "f1-score": 0.5380546650917981, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6762121352462495, + "recall": 0.6744282744282745, + "f1-score": 0.6081579487558598, + "support": 26455.0 + }, + "cross-entropy": 0.6464507795618373, + "roc-auc": 0.6088559366177836, + "score": 0.6088559366177836 + } + }, + "time": "0:01:07.040718", + "diversity": { + "train": { + "mean_centered_corr": -0.007606011815369129, + "mean_pairwise_disagreement": 0.026396666421797922, + "member_std": 0.03116724081337452 + }, + "val": { + "mean_centered_corr": 0.0018892597872763872, + "mean_pairwise_disagreement": 0.09870647301988722, + "member_std": 0.06490844488143921 + }, + "test": { + "mean_centered_corr": 0.01152767427265644, + "mean_pairwise_disagreement": 0.06456551295260973, + "member_std": 0.06369828432798386 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 29 + ], + "head_selection_time": "0:00:04.886969" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-evaluation/0.toml new file mode 100644 index 000000000..5319a963b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-evaluation/0.toml @@ -0,0 +1,46 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.7 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-evaluation/0/report.json new file mode 100644 index 000000000..d3d96f1bb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-evaluation/0/report.json @@ -0,0 +1,251 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.6894886493682861, + "min_features_per_member": 67.0, + "mean_features_per_member": 75.84375, + "max_features_per_member": 84.0, + "min_members_per_feature": 14.0, + "mean_members_per_feature": 22.063636779785156, + "max_members_per_feature": 31.0, + "mean_pairwise_jaccard": 0.5278457608823273, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 428, + "metrics": { + "train": { + "0": { + "precision": 0.7830881310769057, + "recall": 0.9845092006319865, + "f1-score": 0.8723225093416136, + "support": 84179.0 + }, + "1": { + "precision": 0.6284900284900284, + "recall": 0.08767188617756935, + "f1-score": 0.15387834821428573, + "support": 25162.0 + }, + "accuracy": 0.7781253143834427, + "macro avg": { + "precision": 0.705789079783467, + "recall": 0.5360905434047779, + "f1-score": 0.5131004287779497, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7475113807518583, + "recall": 0.7781253143834427, + "f1-score": 0.7069911882243216, + "support": 109341.0 + }, + "cross-entropy": 0.4934543091460455, + "roc-auc": 0.6961052398986505, + "score": 0.6961052398986505 + }, + "val": { + "0": { + "precision": 0.6825136099117702, + "recall": 0.9371697383683464, + "f1-score": 0.7898224080812469, + "support": 15518.0 + }, + "1": { + "precision": 0.6698272942770064, + "recall": 0.22623813336383392, + "f1-score": 0.3382352941176471, + "support": 8743.0 + }, + "accuracy": 0.6809694571534561, + "macro avg": { + "precision": 0.6761704520943883, + "recall": 0.5817039358660901, + "f1-score": 0.564028851099447, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6779418091782992, + "recall": 0.6809694571534561, + "f1-score": 0.627082779154832, + "support": 24261.0 + }, + "cross-entropy": 0.6467973900252932, + "roc-auc": 0.6291199291618959, + "score": 0.6291199291618959 + }, + "test": { + "0": { + "precision": 0.661515493867011, + "recall": 0.9688571090887602, + "f1-score": 0.7862178103869947, + "support": 16922.0 + }, + "1": { + "precision": 0.6846199880311191, + "recall": 0.12000419595090737, + "f1-score": 0.20421278114958943, + "support": 9533.0 + }, + "accuracy": 0.662974862974863, + "macro avg": { + "precision": 0.6730677409490651, + "recall": 0.5444306525198338, + "f1-score": 0.49521529576829204, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6698411465930153, + "recall": 0.662974862974863, + "f1-score": 0.5764936015901629, + "support": 26455.0 + }, + "cross-entropy": 0.6863610680565896, + "roc-auc": 0.5961977102213372, + "score": 0.5961977102213372 + } + }, + "time": "0:01:10.628940", + "diversity": { + "train": { + "mean_centered_corr": -0.015634549781680107, + "mean_pairwise_disagreement": 0.03130809179096706, + "member_std": 0.0355970673263073 + }, + "val": { + "mean_centered_corr": -0.0187542624771595, + "mean_pairwise_disagreement": 0.12205770312369114, + "member_std": 0.07598911970853806 + }, + "test": { + "mean_centered_corr": -0.018549803644418716, + "mean_pairwise_disagreement": 0.08149855811146134, + "member_std": 0.0770004391670227 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-evaluation/1.toml new file mode 100644 index 000000000..503e15153 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-evaluation/1.toml @@ -0,0 +1,46 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 1 +head_selection = true + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.7 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 1 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-evaluation/1/report.json new file mode 100644 index 000000000..aef442869 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-evaluation/1/report.json @@ -0,0 +1,251 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + }, + "mask_stats": { + "keep_rate_actual": 0.703125, + "min_features_per_member": 67.0, + "mean_features_per_member": 77.34375, + "max_features_per_member": 88.0, + "min_members_per_feature": 14.0, + "mean_members_per_feature": 22.5, + "max_members_per_feature": 29.0, + "mean_pairwise_jaccard": 0.5422206532495487, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 428, + "metrics": { + "train": { + "0": { + "precision": 0.7825586551070273, + "recall": 0.9854239180793309, + "f1-score": 0.8723525081501735, + "support": 84179.0 + }, + "1": { + "precision": 0.6326347305389222, + "recall": 0.08397583657896829, + "f1-score": 0.1482702968212757, + "support": 25162.0 + }, + "accuracy": 0.7779789831810574, + "macro avg": { + "precision": 0.7075966928229747, + "recall": 0.5346998773291496, + "f1-score": 0.5103114024857246, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7480575458252148, + "recall": 0.7779789831810574, + "f1-score": 0.7057237357641726, + "support": 109341.0 + }, + "cross-entropy": 0.49245265673530425, + "roc-auc": 0.6989706230822266, + "score": 0.6989706230822266 + }, + "val": { + "0": { + "precision": 0.6779895056614195, + "recall": 0.9492202603428277, + "f1-score": 0.7909998926001504, + "support": 15518.0 + }, + "1": { + "precision": 0.6891518737672584, + "recall": 0.1998169964543063, + "f1-score": 0.3098067033161908, + "support": 8743.0 + }, + "accuracy": 0.6791558468323647, + "macro avg": { + "precision": 0.683570689714339, + "recall": 0.574518628398567, + "f1-score": 0.5504032979581706, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.682012117439555, + "recall": 0.6791558468323647, + "f1-score": 0.6175910449059226, + "support": 24261.0 + }, + "cross-entropy": 0.6597667526471618, + "roc-auc": 0.6253906113125361, + "score": 0.6253906113125361 + }, + "test": { + "0": { + "precision": 0.6585443924298804, + "recall": 0.9726391679470512, + "f1-score": 0.7853513062149589, + "support": 16922.0 + }, + "1": { + "precision": 0.6833105335157319, + "recall": 0.10479387391167523, + "f1-score": 0.18171896316507502, + "support": 9533.0 + }, + "accuracy": 0.6599130599130599, + "macro avg": { + "precision": 0.6709274629728061, + "recall": 0.5387165209293632, + "f1-score": 0.48353513469001697, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6674688159026236, + "recall": 0.6599130599130599, + "f1-score": 0.5678337433234623, + "support": 26455.0 + }, + "cross-entropy": 0.7108761178650036, + "roc-auc": 0.5930931510151916, + "score": 0.5930931510151916 + } + }, + "time": "0:01:12.475731", + "diversity": { + "train": { + "mean_centered_corr": -0.013842575252056122, + "mean_pairwise_disagreement": 0.028755187603386977, + "member_std": 0.03266480937600136 + }, + "val": { + "mean_centered_corr": -0.0060155028477311134, + "mean_pairwise_disagreement": 0.08212387197825795, + "member_std": 0.06719459593296051 + }, + "test": { + "mean_centered_corr": 0.0020387060940265656, + "mean_pairwise_disagreement": 0.052944440041214236, + "member_std": 0.0697987824678421 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-evaluation/2.toml new file mode 100644 index 000000000..fc9553ac2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-evaluation/2.toml @@ -0,0 +1,46 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 2 +head_selection = true + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.7 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 2 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-evaluation/2/report.json new file mode 100644 index 000000000..3bb7bd057 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-evaluation/2/report.json @@ -0,0 +1,251 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + }, + "mask_stats": { + "keep_rate_actual": 0.7090908885002136, + "min_features_per_member": 65.0, + "mean_features_per_member": 78.0, + "max_features_per_member": 92.0, + "min_members_per_feature": 17.0, + "mean_members_per_feature": 22.690908432006836, + "max_members_per_feature": 28.0, + "mean_pairwise_jaccard": 0.5463052047602153, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7830380344909048, + "recall": 0.9843904061583055, + "f1-score": 0.8722447948464243, + "support": 84179.0 + }, + "1": { + "precision": 0.6262798634812287, + "recall": 0.0875129163023607, + "f1-score": 0.15356719436501848, + "support": 25162.0 + }, + "accuracy": 0.7779972745813556, + "macro avg": { + "precision": 0.7046589489860667, + "recall": 0.5359516612303331, + "f1-score": 0.5129059946057214, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7469642003486757, + "recall": 0.7779972745813556, + "f1-score": 0.7068597537062011, + "support": 109341.0 + }, + "cross-entropy": 0.4927953815266221, + "roc-auc": 0.6982500719492171, + "score": 0.6982500719492171 + }, + "val": { + "0": { + "precision": 0.6809063893016345, + "recall": 0.9449671349400696, + "f1-score": 0.7914934959788417, + "support": 15518.0 + }, + "1": { + "precision": 0.686605504587156, + "recall": 0.21399977124556788, + "f1-score": 0.32629926752703176, + "support": 8743.0 + }, + "accuracy": 0.6815465149828943, + "macro avg": { + "precision": 0.6837559469443952, + "recall": 0.5794834530928187, + "f1-score": 0.5588963817529367, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6829601943773246, + "recall": 0.6815465149828943, + "f1-score": 0.6238502356287252, + "support": 24261.0 + }, + "cross-entropy": 0.6743491854283338, + "roc-auc": 0.6220034632459894, + "score": 0.6220034632459894 + }, + "test": { + "0": { + "precision": 0.6608177720540889, + "recall": 0.9703344758302801, + "f1-score": 0.7862101987072061, + "support": 16922.0 + }, + "1": { + "precision": 0.6876166770379589, + "recall": 0.11591314381621735, + "f1-score": 0.19838420107719928, + "support": 9533.0 + }, + "accuracy": 0.6624456624456625, + "macro avg": { + "precision": 0.6742172245460238, + "recall": 0.5431238098232487, + "f1-score": 0.49229719989220266, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6704746974447988, + "recall": 0.6624456624456625, + "f1-score": 0.5743884169870453, + "support": 26455.0 + }, + "cross-entropy": 0.7345531868914768, + "roc-auc": 0.5900390978219552, + "score": 0.5900390978219552 + } + }, + "time": "0:01:07.040718", + "diversity": { + "train": { + "mean_centered_corr": -0.007606011815369129, + "mean_pairwise_disagreement": 0.026396666421797922, + "member_std": 0.03116724081337452 + }, + "val": { + "mean_centered_corr": 0.0018892597872763872, + "mean_pairwise_disagreement": 0.09870647301988722, + "member_std": 0.06490844488143921 + }, + "test": { + "mean_centered_corr": 0.01152767427265644, + "mean_pairwise_disagreement": 0.06456551295260973, + "member_std": 0.06369828432798386 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..d21da633b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,255 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.6894886493682861, + "min_features_per_member": 67.0, + "mean_features_per_member": 75.84375, + "max_features_per_member": 84.0, + "min_members_per_feature": 14.0, + "mean_members_per_feature": 22.063636779785156, + "max_members_per_feature": 31.0, + "mean_pairwise_jaccard": 0.5278457608823273, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 428, + "metrics": { + "train": { + "0": { + "precision": 0.7894966160281175, + "recall": 0.9686501383955618, + "f1-score": 0.8699455883921904, + "support": 84179.0 + }, + "1": { + "precision": 0.5645214521452145, + "recall": 0.13595898577219617, + "f1-score": 0.21914034975337904, + "support": 25162.0 + }, + "accuracy": 0.7770278303655537, + "macro avg": { + "precision": 0.677009034086666, + "recall": 0.552304562083879, + "f1-score": 0.5445429690727848, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7377244073084094, + "recall": 0.7770278303655537, + "f1-score": 0.7201796139212255, + "support": 109341.0 + }, + "cross-entropy": 0.5002029863194409, + "roc-auc": 0.6821879786641952, + "score": 0.6821879786641952 + }, + "val": { + "0": { + "precision": 0.6961402811584124, + "recall": 0.9030802938523006, + "f1-score": 0.7862212123762238, + "support": 15518.0 + }, + "1": { + "precision": 0.6358353510895883, + "recall": 0.30035456936978155, + "f1-score": 0.4079857065175173, + "support": 8743.0 + }, + "accuracy": 0.6858744487036808, + "macro avg": { + "precision": 0.6659878161240004, + "recall": 0.6017174316110411, + "f1-score": 0.5971034594468705, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6744080358432264, + "recall": 0.6858744487036808, + "f1-score": 0.6499154942391862, + "support": 24261.0 + }, + "cross-entropy": 0.6154470541736884, + "roc-auc": 0.6547032702847417, + "score": 0.6547032702847417 + }, + "test": { + "0": { + "precision": 0.6733296707940667, + "recall": 0.9415553717054722, + "f1-score": 0.7851669335961562, + "support": 16922.0 + }, + "1": { + "precision": 0.6457736389684814, + "recall": 0.18913248714990036, + "f1-score": 0.2925760649087221, + "support": 9533.0 + }, + "accuracy": 0.6704214704214704, + "macro avg": { + "precision": 0.6595516548812741, + "recall": 0.5653439294276863, + "f1-score": 0.5388714992524392, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6633999164416454, + "recall": 0.6704214704214704, + "f1-score": 0.6076629172968816, + "support": 26455.0 + }, + "cross-entropy": 0.638677137478482, + "roc-auc": 0.6118564029158264, + "score": 0.6118564029158264 + } + }, + "time": "0:01:10.628940", + "diversity": { + "train": { + "mean_centered_corr": -0.015634549781680107, + "mean_pairwise_disagreement": 0.03130809179096706, + "member_std": 0.0355970673263073 + }, + "val": { + "mean_centered_corr": -0.0187542624771595, + "mean_pairwise_disagreement": 0.12205770312369114, + "member_std": 0.07598911970853806 + }, + "test": { + "mean_centered_corr": -0.018549803644418716, + "mean_pairwise_disagreement": 0.08149855811146134, + "member_std": 0.0770004391670227 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 16 + ], + "head_selection_time": "0:00:11.162732" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-greedy-heads-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-greedy-heads-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-greedy-heads-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-greedy-heads-evaluation/1/report.json new file mode 100644 index 000000000..ef997fb74 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-greedy-heads-evaluation/1/report.json @@ -0,0 +1,256 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + }, + "mask_stats": { + "keep_rate_actual": 0.703125, + "min_features_per_member": 67.0, + "mean_features_per_member": 77.34375, + "max_features_per_member": 88.0, + "min_members_per_feature": 14.0, + "mean_members_per_feature": 22.5, + "max_members_per_feature": 29.0, + "mean_pairwise_jaccard": 0.5422206532495487, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 428, + "metrics": { + "train": { + "0": { + "precision": 0.7910205588589639, + "recall": 0.9671652074745483, + "f1-score": 0.870269316900317, + "support": 84179.0 + }, + "1": { + "precision": 0.569269128876422, + "recall": 0.14517923853429776, + "f1-score": 0.23135628107286488, + "support": 25162.0 + }, + "accuracy": 0.7780064202815047, + "macro avg": { + "precision": 0.6801448438676929, + "recall": 0.556172223004423, + "f1-score": 0.550812798986591, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7399902090247689, + "recall": 0.7780064202815047, + "f1-score": 0.7232400249833751, + "support": 109341.0 + }, + "cross-entropy": 0.49719324451950025, + "roc-auc": 0.6874785943212433, + "score": 0.6874785943212433 + }, + "val": { + "0": { + "precision": 0.6957908163265306, + "recall": 0.9139708725351205, + "f1-score": 0.7900952593170297, + "support": 15518.0 + }, + "1": { + "precision": 0.6556615940159918, + "recall": 0.2907468832208624, + "f1-score": 0.4028526148969889, + "support": 8743.0 + }, + "accuracy": 0.6893780140966984, + "macro avg": { + "precision": 0.6757262051712611, + "recall": 0.6023588778779915, + "f1-score": 0.5964739371070094, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6813293435652659, + "recall": 0.6893780140966984, + "f1-score": 0.650543615107623, + "support": 24261.0 + }, + "cross-entropy": 0.6160387174694619, + "roc-auc": 0.6527163070467052, + "score": 0.6527163070467052 + }, + "test": { + "0": { + "precision": 0.6715682178962319, + "recall": 0.9500059094669661, + "f1-score": 0.7868820362212433, + "support": 16922.0 + }, + "1": { + "precision": 0.66388557806913, + "recall": 0.1752858491555649, + "f1-score": 0.277344398340249, + "support": 9533.0 + }, + "accuracy": 0.6708372708372708, + "macro avg": { + "precision": 0.6677268979826809, + "recall": 0.5626458793112654, + "f1-score": 0.5321132172807461, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6687997958409773, + "recall": 0.6708372708372708, + "f1-score": 0.6032712895979389, + "support": 26455.0 + }, + "cross-entropy": 0.6453489859863742, + "roc-auc": 0.6065300223672054, + "score": 0.6065300223672054 + } + }, + "time": "0:01:12.475731", + "diversity": { + "train": { + "mean_centered_corr": -0.013842575252056122, + "mean_pairwise_disagreement": 0.028755187603386977, + "member_std": 0.03266480937600136 + }, + "val": { + "mean_centered_corr": -0.0060155028477311134, + "mean_pairwise_disagreement": 0.08212387197825795, + "member_std": 0.06719459593296051 + }, + "test": { + "mean_centered_corr": 0.0020387060940265656, + "mean_pairwise_disagreement": 0.052944440041214236, + "member_std": 0.0697987824678421 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 16, + 26 + ], + "head_selection_time": "0:00:15.998123" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-greedy-heads-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-greedy-heads-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-greedy-heads-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-greedy-heads-evaluation/2/report.json new file mode 100644 index 000000000..898ed4405 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-greedy-heads-evaluation/2/report.json @@ -0,0 +1,255 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + }, + "mask_stats": { + "keep_rate_actual": 0.7090908885002136, + "min_features_per_member": 65.0, + "mean_features_per_member": 78.0, + "max_features_per_member": 92.0, + "min_members_per_feature": 17.0, + "mean_members_per_feature": 22.690908432006836, + "max_members_per_feature": 28.0, + "mean_pairwise_jaccard": 0.5463052047602153, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7825240884186662, + "recall": 0.9840696610793666, + "f1-score": 0.8717999989475844, + "support": 84179.0 + }, + "1": { + "precision": 0.6147658718758977, + "recall": 0.08504888323662665, + "f1-score": 0.14942568864993191, + "support": 25162.0 + }, + "accuracy": 0.7771833072680879, + "macro avg": { + "precision": 0.6986449801472819, + "recall": 0.5345592721579966, + "f1-score": 0.5106128437987582, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7439188786195136, + "recall": 0.7771833072680879, + "f1-score": 0.7055642557615013, + "support": 109341.0 + }, + "cross-entropy": 0.49927567921826405, + "roc-auc": 0.6841046410049181, + "score": 0.6841046410049181 + }, + "val": { + "0": { + "precision": 0.6951219512195121, + "recall": 0.9256347467457147, + "f1-score": 0.7939859598695483, + "support": 15518.0 + }, + "1": { + "precision": 0.6791770920211287, + "recall": 0.27942353883106485, + "f1-score": 0.3959481361426256, + "support": 8743.0 + }, + "accuracy": 0.6927579242405507, + "macro avg": { + "precision": 0.6871495216203205, + "recall": 0.6025291427883898, + "f1-score": 0.5949670480060869, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.689375860622609, + "recall": 0.6927579242405507, + "f1-score": 0.6505440286694953, + "support": 24261.0 + }, + "cross-entropy": 0.6149192758236253, + "roc-auc": 0.6546592087434608, + "score": 0.6546592087434608 + }, + "test": { + "0": { + "precision": 0.6737775546910947, + "recall": 0.9518969388961116, + "f1-score": 0.7890470009062186, + "support": 16922.0 + }, + "1": { + "precision": 0.6805337519623234, + "recall": 0.18189447183467952, + "f1-score": 0.2870623292773777, + "support": 9533.0 + }, + "accuracy": 0.6744282744282745, + "macro avg": { + "precision": 0.677155653326709, + "recall": 0.5668957053653956, + "f1-score": 0.5380546650917981, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6762121352462495, + "recall": 0.6744282744282745, + "f1-score": 0.6081579487558598, + "support": 26455.0 + }, + "cross-entropy": 0.6464507795618373, + "roc-auc": 0.6088559366177836, + "score": 0.6088559366177836 + } + }, + "time": "0:01:07.040718", + "diversity": { + "train": { + "mean_centered_corr": -0.007606011815369129, + "mean_pairwise_disagreement": 0.026396666421797922, + "member_std": 0.03116724081337452 + }, + "val": { + "mean_centered_corr": 0.0018892597872763872, + "mean_pairwise_disagreement": 0.09870647301988722, + "member_std": 0.06490844488143921 + }, + "test": { + "mean_centered_corr": 0.01152767427265644, + "mean_pairwise_disagreement": 0.06456551295260973, + "member_std": 0.06369828432798386 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 29 + ], + "head_selection_time": "0:00:11.198245" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..16776bce5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/0/report.json @@ -0,0 +1,228 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1951424, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.779700580278046, + "recall": 0.9880492759476829, + "f1-score": 0.8715968394357931, + "support": 84179.0 + }, + "1": { + "precision": 0.6229385307346327, + "recall": 0.06605198314919322, + "f1-score": 0.11943945382680561, + "support": 25162.0 + }, + "accuracy": 0.7758754721467702, + "macro avg": { + "precision": 0.7013195555063394, + "recall": 0.5270506295484381, + "f1-score": 0.4955181466312994, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7436258535917035, + "recall": 0.7758754721467702, + "f1-score": 0.6985072926354772, + "support": 109341.0 + }, + "cross-entropy": 0.5001462491291722, + "roc-auc": 0.6828783854988578, + "score": 0.6828783854988578 + }, + "val": { + "0": { + "precision": 0.6734214920792526, + "recall": 0.9615285474932337, + "f1-score": 0.7920902455209025, + "support": 15518.0 + }, + "1": { + "precision": 0.716254752851711, + "recall": 0.17236646460025162, + "f1-score": 0.2778648474232507, + "support": 8743.0 + }, + "accuracy": 0.677136144429331, + "macro avg": { + "precision": 0.6948381224654818, + "recall": 0.5669475060467426, + "f1-score": 0.5349775464720765, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.688857426250705, + "recall": 0.677136144429331, + "f1-score": 0.6067774943742981, + "support": 24261.0 + }, + "cross-entropy": 0.6457686779603163, + "roc-auc": 0.6396541901648656, + "score": 0.6396541901648656 + }, + "test": { + "0": { + "precision": 0.6569502280388657, + "recall": 0.9789032029310956, + "f1-score": 0.7862448679307972, + "support": 16922.0 + }, + "1": { + "precision": 0.7120967741935483, + "recall": 0.09262561628028952, + "f1-score": 0.16392833936693585, + "support": 9533.0 + }, + "accuracy": 0.6595350595350595, + "macro avg": { + "precision": 0.684523501116207, + "recall": 0.5357644096056925, + "f1-score": 0.4750866036488665, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.67682216243662, + "recall": 0.6595350595350595, + "f1-score": 0.5619945006354167, + "support": 26455.0 + }, + "cross-entropy": 0.6822343746621596, + "roc-auc": 0.6015418321886687, + "score": 0.6015418321886687 + } + }, + "time": "0:01:18.218817", + "diversity": { + "train": { + "mean_centered_corr": -0.016645358875393867, + "mean_pairwise_disagreement": 0.024673918911862297, + "member_std": 0.027964944019913673 + }, + "val": { + "mean_centered_corr": 0.002000282984226942, + "mean_pairwise_disagreement": 0.06748003233651247, + "member_std": 0.05873498693108559 + }, + "test": { + "mean_centered_corr": 0.008286470547318459, + "mean_pairwise_disagreement": 0.04453758969888002, + "member_std": 0.06023459881544113 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 11 + ], + "head_selection_time": "0:00:04.887253" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/1/report.json new file mode 100644 index 000000000..c19a0f6c2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/1/report.json @@ -0,0 +1,228 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1951424, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 428, + "metrics": { + "train": { + "0": { + "precision": 0.7922289232927696, + "recall": 0.9659416243956331, + "f1-score": 0.8705035971223022, + "support": 84179.0 + }, + "1": { + "precision": 0.5723448687350835, + "recall": 0.15249185279389554, + "f1-score": 0.24082093767652044, + "support": 25162.0 + }, + "accuracy": 0.7787472219935797, + "macro avg": { + "precision": 0.6822868960139266, + "recall": 0.5592167385947644, + "f1-score": 0.5556622673994113, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7416283015609353, + "recall": 0.7787472219935797, + "f1-score": 0.72559843732886, + "support": 109341.0 + }, + "cross-entropy": 0.49316481374570875, + "roc-auc": 0.695882539918458, + "score": 0.695882539918458 + }, + "val": { + "0": { + "precision": 0.6824891054777189, + "recall": 0.9385874468359324, + "f1-score": 0.7903090154372067, + "support": 15518.0 + }, + "1": { + "precision": 0.6736301369863014, + "recall": 0.22497998398718974, + "f1-score": 0.3373060104604304, + "support": 8743.0 + }, + "accuracy": 0.6814228597337291, + "macro avg": { + "precision": 0.6780596212320101, + "recall": 0.581783715411561, + "f1-score": 0.5638075129488185, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.679296575840834, + "recall": 0.6814228597337291, + "f1-score": 0.6270591381645486, + "support": 24261.0 + }, + "cross-entropy": 0.6646895058244983, + "roc-auc": 0.6311484553024557, + "score": 0.6311484553024557 + }, + "test": { + "0": { + "precision": 0.6616923449221586, + "recall": 0.969507150455029, + "f1-score": 0.7865567168472528, + "support": 16922.0 + }, + "1": { + "precision": 0.6893437688139675, + "recall": 0.12010909472359173, + "f1-score": 0.20457387886367698, + "support": 9533.0 + }, + "accuracy": 0.6634284634284634, + "macro avg": { + "precision": 0.675518056868063, + "recall": 0.5448081225893103, + "f1-score": 0.4955652978554649, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6716564735919985, + "recall": 0.6634284634284634, + "f1-score": 0.5768405046190378, + "support": 26455.0 + }, + "cross-entropy": 0.7219351511377476, + "roc-auc": 0.5960986415689524, + "score": 0.5960986415689524 + } + }, + "time": "0:01:28.118874", + "diversity": { + "train": { + "mean_centered_corr": -0.010014320723712444, + "mean_pairwise_disagreement": 0.02492483562079095, + "member_std": 0.024375617504119873 + }, + "val": { + "mean_centered_corr": 0.02006583847105503, + "mean_pairwise_disagreement": 0.06821390297184782, + "member_std": 0.04597385972738266 + }, + "test": { + "mean_centered_corr": 0.03903863951563835, + "mean_pairwise_disagreement": 0.03914178672243188, + "member_std": 0.04212528467178345 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 15 + ], + "head_selection_time": "0:00:04.875768" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/2/report.json new file mode 100644 index 000000000..d4aba5fe6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/2/report.json @@ -0,0 +1,228 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1951424, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7856289713821605, + "recall": 0.9767281626058756, + "f1-score": 0.8708177552771217, + "support": 84179.0 + }, + "1": { + "precision": 0.5819462227912933, + "recall": 0.10837771242349575, + "f1-score": 0.18272581077459127, + "support": 25162.0 + }, + "accuracy": 0.7768997905634666, + "macro avg": { + "precision": 0.6837875970867269, + "recall": 0.5425529375146857, + "f1-score": 0.5267717830258565, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7387566607206209, + "recall": 0.7768997905634666, + "f1-score": 0.7124712109106656, + "support": 109341.0 + }, + "cross-entropy": 0.49964959969501876, + "roc-auc": 0.6794902287315214, + "score": 0.6794902287315214 + }, + "val": { + "0": { + "precision": 0.6822470605312817, + "recall": 0.9086222451346823, + "f1-score": 0.7793284510156142, + "support": 15518.0 + }, + "1": { + "precision": 0.6054535336672231, + "recall": 0.24888482214342902, + "f1-score": 0.352759990273162, + "support": 8743.0 + }, + "accuracy": 0.6708709451382878, + "macro avg": { + "precision": 0.6438502970992523, + "recall": 0.5787535336390557, + "f1-score": 0.5660442206443881, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6545727764798219, + "recall": 0.6708709451382878, + "f1-score": 0.6256048595613766, + "support": 24261.0 + }, + "cross-entropy": 0.6436639930347295, + "roc-auc": 0.633681175787757, + "score": 0.633681175787757 + }, + "test": { + "0": { + "precision": 0.6632580953568608, + "recall": 0.9429145491076705, + "f1-score": 0.7787403304131384, + "support": 16922.0 + }, + "1": { + "precision": 0.5971643035863219, + "recall": 0.15021504248400294, + "f1-score": 0.24004693655183976, + "support": 9533.0 + }, + "accuracy": 0.6572670572670573, + "macro avg": { + "precision": 0.6302111994715913, + "recall": 0.5465647957958367, + "f1-score": 0.5093936334824891, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6394413455194559, + "recall": 0.6572670572670573, + "f1-score": 0.5846233724210854, + "support": 26455.0 + }, + "cross-entropy": 0.6816832612823018, + "roc-auc": 0.597700306723218, + "score": 0.597700306723218 + } + }, + "time": "0:01:22.843981", + "diversity": { + "train": { + "mean_centered_corr": -0.01568707637488842, + "mean_pairwise_disagreement": 0.023222997836599382, + "member_std": 0.024020660668611526 + }, + "val": { + "mean_centered_corr": -0.00023190633510239422, + "mean_pairwise_disagreement": 0.07023343917158961, + "member_std": 0.050259560346603394 + }, + "test": { + "mean_centered_corr": 0.006958201061934233, + "mean_pairwise_disagreement": 0.04134691594369014, + "member_std": 0.04727793484926224 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 20 + ], + "head_selection_time": "0:00:04.821733" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-evaluation/0.toml new file mode 100644 index 000000000..2990735ec --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-evaluation/0/report.json new file mode 100644 index 000000000..1b45b9b2d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-evaluation/0/report.json @@ -0,0 +1,224 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1951424, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7813988964010622, + "recall": 0.9857921809477423, + "f1-score": 0.8717754770796893, + "support": 84179.0 + }, + "1": { + "precision": 0.6194718421889914, + "recall": 0.0773785867578094, + "f1-score": 0.1375728669846317, + "support": 25162.0 + }, + "accuracy": 0.7767443136609323, + "macro avg": { + "precision": 0.7004353692950268, + "recall": 0.5315853838527759, + "f1-score": 0.5046741720321605, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.744135577626914, + "recall": 0.7767443136609323, + "f1-score": 0.7028177569636135, + "support": 109341.0 + }, + "cross-entropy": 0.4956556906392919, + "roc-auc": 0.6916462356019382, + "score": 0.6916462356019382 + }, + "val": { + "0": { + "precision": 0.6749512449544197, + "recall": 0.9590153370279675, + "f1-score": 0.7922911065562861, + "support": 15518.0 + }, + "1": { + "precision": 0.7124773960216998, + "recall": 0.18025849250829235, + "f1-score": 0.2877225011410315, + "support": 8743.0 + }, + "accuracy": 0.6783726969209843, + "macro avg": { + "precision": 0.6937143204880598, + "recall": 0.5696369147681299, + "f1-score": 0.5400068038486587, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6884746421260627, + "recall": 0.6783726969209843, + "f1-score": 0.6104583990361686, + "support": 24261.0 + }, + "cross-entropy": 0.6750056416383268, + "roc-auc": 0.6227001707049361, + "score": 0.6227001707049361 + }, + "test": { + "0": { + "precision": 0.6569412510901451, + "recall": 0.9793168656187212, + "f1-score": 0.7863718325899213, + "support": 16922.0 + }, + "1": { + "precision": 0.7152156224572823, + "recall": 0.09220602118955208, + "f1-score": 0.163352536703215, + "support": 9533.0 + }, + "accuracy": 0.6596484596484596, + "macro avg": { + "precision": 0.6860784367737137, + "recall": 0.5357614434041367, + "f1-score": 0.47486218464656815, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6779402902979667, + "recall": 0.6596484596484596, + "f1-score": 0.561868224663708, + "support": 26455.0 + }, + "cross-entropy": 0.7361612281771354, + "roc-auc": 0.5911765632808944, + "score": 0.5911765632808944 + } + }, + "time": "0:01:18.218817", + "diversity": { + "train": { + "mean_centered_corr": -0.016645358875393867, + "mean_pairwise_disagreement": 0.024673918911862297, + "member_std": 0.027964944019913673 + }, + "val": { + "mean_centered_corr": 0.002000282984226942, + "mean_pairwise_disagreement": 0.06748003233651247, + "member_std": 0.05873498693108559 + }, + "test": { + "mean_centered_corr": 0.008286470547318459, + "mean_pairwise_disagreement": 0.04453758969888002, + "member_std": 0.06023459881544113 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-evaluation/1.toml new file mode 100644 index 000000000..1cf01543f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-evaluation/1.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 1 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-evaluation/1/report.json new file mode 100644 index 000000000..434ecb008 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-evaluation/1/report.json @@ -0,0 +1,224 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1951424, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 428, + "metrics": { + "train": { + "0": { + "precision": 0.7881281081392446, + "recall": 0.9752194727901258, + "f1-score": 0.8717485836859739, + "support": 84179.0 + }, + "1": { + "precision": 0.5972195404518247, + "recall": 0.12292345600508704, + "f1-score": 0.20388253518341518, + "support": 25162.0 + }, + "accuracy": 0.7790856128990955, + "macro avg": { + "precision": 0.6926738242955346, + "recall": 0.5490714643976065, + "f1-score": 0.5378155594346945, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7441954444526964, + "recall": 0.7790856128990955, + "f1-score": 0.7180565055778407, + "support": 109341.0 + }, + "cross-entropy": 0.4903386782360664, + "roc-auc": 0.7032463632265398, + "score": 0.7032463632265398 + }, + "val": { + "0": { + "precision": 0.6736268724466636, + "recall": 0.9563088026807578, + "f1-score": 0.790454884414616, + "support": 15518.0 + }, + "1": { + "precision": 0.6961004034065441, + "recall": 0.17762781653894544, + "f1-score": 0.2830326225624203, + "support": 8743.0 + }, + "accuracy": 0.6756934998557356, + "macro avg": { + "precision": 0.6848636379266039, + "recall": 0.5669683096098517, + "f1-score": 0.5367437534885181, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6817257175553663, + "recall": 0.6756934998557356, + "f1-score": 0.6075937972634785, + "support": 24261.0 + }, + "cross-entropy": 0.715360677146401, + "roc-auc": 0.6203498803314189, + "score": 0.6203498803314189 + }, + "test": { + "0": { + "precision": 0.655632548010295, + "recall": 0.97848954024347, + "f1-score": 0.785167271261588, + "support": 16922.0 + }, + "1": { + "precision": 0.6966666666666667, + "recall": 0.08769537396412462, + "f1-score": 0.15578123544209446, + "support": 9533.0 + }, + "accuracy": 0.6574938574938575, + "macro avg": { + "precision": 0.6761496073384808, + "recall": 0.5330924571037973, + "f1-score": 0.47047425335184123, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6704191007659627, + "recall": 0.6574938574938575, + "f1-score": 0.5583694228598782, + "support": 26455.0 + }, + "cross-entropy": 0.8001816812510966, + "roc-auc": 0.5897023859034299, + "score": 0.5897023859034299 + } + }, + "time": "0:01:28.118874", + "diversity": { + "train": { + "mean_centered_corr": -0.010014320723712444, + "mean_pairwise_disagreement": 0.02492483562079095, + "member_std": 0.024375617504119873 + }, + "val": { + "mean_centered_corr": 0.02006583847105503, + "mean_pairwise_disagreement": 0.06821390297184782, + "member_std": 0.04597385972738266 + }, + "test": { + "mean_centered_corr": 0.03903863951563835, + "mean_pairwise_disagreement": 0.03914178672243188, + "member_std": 0.04212528467178345 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-evaluation/2.toml new file mode 100644 index 000000000..d7e69509b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-evaluation/2.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 2 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-evaluation/2/report.json new file mode 100644 index 000000000..acd01dc37 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-evaluation/2/report.json @@ -0,0 +1,224 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1951424, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.786029867837206, + "recall": 0.9785338386058281, + "f1-score": 0.8717813032480659, + "support": 84179.0 + }, + "1": { + "precision": 0.6025076990761109, + "recall": 0.10885462204912169, + "f1-score": 0.1843947758179615, + "support": 25162.0 + }, + "accuracy": 0.7783996853879148, + "macro avg": { + "precision": 0.6942687834566584, + "recall": 0.5436942303274749, + "f1-score": 0.5280880395330136, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7437969926086396, + "recall": 0.7783996853879148, + "f1-score": 0.713597092355571, + "support": 109341.0 + }, + "cross-entropy": 0.491904955768715, + "roc-auc": 0.699991829704937, + "score": 0.699991829704937 + }, + "val": { + "0": { + "precision": 0.6761661574939584, + "recall": 0.9555999484469648, + "f1-score": 0.7919570616037811, + "support": 15518.0 + }, + "1": { + "precision": 0.7042918454935623, + "recall": 0.18769301155209883, + "f1-score": 0.29639664047683556, + "support": 8743.0 + }, + "accuracy": 0.6788673179176457, + "macro avg": { + "precision": 0.6902290014937603, + "recall": 0.5716464799995318, + "f1-score": 0.5441768510403083, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6863018852125411, + "recall": 0.6788673179176457, + "f1-score": 0.6133706570073966, + "support": 24261.0 + }, + "cross-entropy": 0.7034186632975251, + "roc-auc": 0.6211526583224122, + "score": 0.6211526583224122 + }, + "test": { + "0": { + "precision": 0.6576497732516509, + "recall": 0.9769530788322893, + "f1-score": 0.7861150737042321, + "support": 16922.0 + }, + "1": { + "precision": 0.7038724373576309, + "recall": 0.09724116227840135, + "f1-score": 0.17087557603686637, + "support": 9533.0 + }, + "accuracy": 0.65995085995086, + "macro avg": { + "precision": 0.680761105304641, + "recall": 0.5370971205553453, + "f1-score": 0.47849532487054924, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6743060067395475, + "recall": 0.65995085995086, + "f1-score": 0.5644148986423158, + "support": 26455.0 + }, + "cross-entropy": 0.781592470979893, + "roc-auc": 0.5892307660549951, + "score": 0.5892307660549951 + } + }, + "time": "0:01:22.843981", + "diversity": { + "train": { + "mean_centered_corr": -0.01568707637488842, + "mean_pairwise_disagreement": 0.023222997836599382, + "member_std": 0.024020660668611526 + }, + "val": { + "mean_centered_corr": -0.00023190633510239422, + "mean_pairwise_disagreement": 0.07023343917158961, + "member_std": 0.050259560346603394 + }, + "test": { + "mean_centered_corr": 0.006958201061934233, + "mean_pairwise_disagreement": 0.04134691594369014, + "member_std": 0.04727793484926224 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..ec841c0fe --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,228 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1951424, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.779700580278046, + "recall": 0.9880492759476829, + "f1-score": 0.8715968394357931, + "support": 84179.0 + }, + "1": { + "precision": 0.6229385307346327, + "recall": 0.06605198314919322, + "f1-score": 0.11943945382680561, + "support": 25162.0 + }, + "accuracy": 0.7758754721467702, + "macro avg": { + "precision": 0.7013195555063394, + "recall": 0.5270506295484381, + "f1-score": 0.4955181466312994, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7436258535917035, + "recall": 0.7758754721467702, + "f1-score": 0.6985072926354772, + "support": 109341.0 + }, + "cross-entropy": 0.5001462491291722, + "roc-auc": 0.6828783854988578, + "score": 0.6828783854988578 + }, + "val": { + "0": { + "precision": 0.6734214920792526, + "recall": 0.9615285474932337, + "f1-score": 0.7920902455209025, + "support": 15518.0 + }, + "1": { + "precision": 0.716254752851711, + "recall": 0.17236646460025162, + "f1-score": 0.2778648474232507, + "support": 8743.0 + }, + "accuracy": 0.677136144429331, + "macro avg": { + "precision": 0.6948381224654818, + "recall": 0.5669475060467426, + "f1-score": 0.5349775464720765, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.688857426250705, + "recall": 0.677136144429331, + "f1-score": 0.6067774943742981, + "support": 24261.0 + }, + "cross-entropy": 0.6457686779603163, + "roc-auc": 0.6396541901648656, + "score": 0.6396541901648656 + }, + "test": { + "0": { + "precision": 0.6569502280388657, + "recall": 0.9789032029310956, + "f1-score": 0.7862448679307972, + "support": 16922.0 + }, + "1": { + "precision": 0.7120967741935483, + "recall": 0.09262561628028952, + "f1-score": 0.16392833936693585, + "support": 9533.0 + }, + "accuracy": 0.6595350595350595, + "macro avg": { + "precision": 0.684523501116207, + "recall": 0.5357644096056925, + "f1-score": 0.4750866036488665, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.67682216243662, + "recall": 0.6595350595350595, + "f1-score": 0.5619945006354167, + "support": 26455.0 + }, + "cross-entropy": 0.6822343746621596, + "roc-auc": 0.6015418321886687, + "score": 0.6015418321886687 + } + }, + "time": "0:01:18.218817", + "diversity": { + "train": { + "mean_centered_corr": -0.016645358875393867, + "mean_pairwise_disagreement": 0.024673918911862297, + "member_std": 0.027964944019913673 + }, + "val": { + "mean_centered_corr": 0.002000282984226942, + "mean_pairwise_disagreement": 0.06748003233651247, + "member_std": 0.05873498693108559 + }, + "test": { + "mean_centered_corr": 0.008286470547318459, + "mean_pairwise_disagreement": 0.04453758969888002, + "member_std": 0.06023459881544113 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 11 + ], + "head_selection_time": "0:00:11.224192" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-greedy-heads-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-greedy-heads-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-greedy-heads-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-greedy-heads-evaluation/1/report.json new file mode 100644 index 000000000..dd3929da1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-greedy-heads-evaluation/1/report.json @@ -0,0 +1,232 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1951424, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 428, + "metrics": { + "train": { + "0": { + "precision": 0.7934275978222552, + "recall": 0.9625678613430904, + "f1-score": 0.869851800561451, + "support": 84179.0 + }, + "1": { + "precision": 0.5633919911320493, + "recall": 0.16159287814959064, + "f1-score": 0.25115043701164336, + "support": 25162.0 + }, + "accuracy": 0.7782442084853806, + "macro avg": { + "precision": 0.6784097944771523, + "recall": 0.5620803697463405, + "f1-score": 0.5605011187865472, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7404908592197277, + "recall": 0.7782442084853806, + "f1-score": 0.727473701681431, + "support": 109341.0 + }, + "cross-entropy": 0.4936760184787449, + "roc-auc": 0.6962309860821628, + "score": 0.6962309860821628 + }, + "val": { + "0": { + "precision": 0.6906453952139231, + "recall": 0.9206083258151824, + "f1-score": 0.7892163632848106, + "support": 15518.0 + }, + "1": { + "precision": 0.6554809843400448, + "recall": 0.2681001944412673, + "f1-score": 0.3805503693481614, + "support": 8743.0 + }, + "accuracy": 0.6854622645397964, + "macro avg": { + "precision": 0.6730631897769839, + "recall": 0.5943542601282248, + "f1-score": 0.584883366316486, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6779731045305085, + "recall": 0.6854622645397964, + "f1-score": 0.641944330599096, + "support": 24261.0 + }, + "cross-entropy": 0.6564983741679696, + "roc-auc": 0.6323603724914644, + "score": 0.6323603724914644 + }, + "test": { + "0": { + "precision": 0.6652895942408377, + "recall": 0.9611748020328567, + "f1-score": 0.7863185883490452, + "support": 16922.0 + }, + "1": { + "precision": 0.672645739910314, + "recall": 0.14161334312388546, + "f1-score": 0.2339688041594454, + "support": 9533.0 + }, + "accuracy": 0.6658476658476659, + "macro avg": { + "precision": 0.6689676670755758, + "recall": 0.551394072578371, + "f1-score": 0.5101436962542453, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6679403648576253, + "recall": 0.6658476658476659, + "f1-score": 0.5872805806877542, + "support": 26455.0 + }, + "cross-entropy": 0.7071797390975189, + "roc-auc": 0.5968416363152236, + "score": 0.5968416363152236 + } + }, + "time": "0:01:28.118874", + "diversity": { + "train": { + "mean_centered_corr": -0.010014320723712444, + "mean_pairwise_disagreement": 0.02492483562079095, + "member_std": 0.024375617504119873 + }, + "val": { + "mean_centered_corr": 0.02006583847105503, + "mean_pairwise_disagreement": 0.06821390297184782, + "member_std": 0.04597385972738266 + }, + "test": { + "mean_centered_corr": 0.03903863951563835, + "mean_pairwise_disagreement": 0.03914178672243188, + "member_std": 0.04212528467178345 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 15, + 10, + 9, + 20, + 9 + ], + "head_selection_time": "0:00:30.228414" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-greedy-heads-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-greedy-heads-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-greedy-heads-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-greedy-heads-evaluation/2/report.json new file mode 100644 index 000000000..174679548 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-greedy-heads-evaluation/2/report.json @@ -0,0 +1,229 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1951424, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7871367345177713, + "recall": 0.9752551111322302, + "f1-score": 0.8711560092531675, + "support": 84179.0 + }, + "1": { + "precision": 0.5870340999206979, + "recall": 0.11767745012320166, + "f1-score": 0.19605376415281733, + "support": 25162.0 + }, + "accuracy": 0.7779058175798649, + "macro avg": { + "precision": 0.6870854172192347, + "recall": 0.546466280627716, + "f1-score": 0.5336048867029924, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7410882943925524, + "recall": 0.7779058175798649, + "f1-score": 0.7157987078637984, + "support": 109341.0 + }, + "cross-entropy": 0.4982793043881891, + "roc-auc": 0.6827272079405879, + "score": 0.6827272079405879 + }, + "val": { + "0": { + "precision": 0.689112461400538, + "recall": 0.89160974352365, + "f1-score": 0.7773907180582088, + "support": 15518.0 + }, + "1": { + "precision": 0.5978962467128854, + "recall": 0.2860574173624614, + "f1-score": 0.3869719944298313, + "support": 8743.0 + }, + "accuracy": 0.6733852685379828, + "macro avg": { + "precision": 0.6435043540567117, + "recall": 0.5888335804430557, + "f1-score": 0.5821813562440201, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.656240635630201, + "recall": 0.6733852685379828, + "f1-score": 0.6366945018806851, + "support": 24261.0 + }, + "cross-entropy": 0.6339658157770617, + "roc-auc": 0.6351641473729864, + "score": 0.6351641473729864 + }, + "test": { + "0": { + "precision": 0.6670301098716765, + "recall": 0.9399598156246307, + "f1-score": 0.7803178963893249, + "support": 16922.0 + }, + "1": { + "precision": 0.6105787658106554, + "recall": 0.16710374488618482, + "f1-score": 0.2623949925877121, + "support": 9533.0 + }, + "accuracy": 0.6614628614628615, + "macro avg": { + "precision": 0.6388044378411659, + "recall": 0.5535317802554077, + "f1-score": 0.5213564444885185, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6466879944706667, + "recall": 0.6614628614628615, + "f1-score": 0.5936855379716052, + "support": 26455.0 + }, + "cross-entropy": 0.6638257558023992, + "roc-auc": 0.5998596611627065, + "score": 0.5998596611627065 + } + }, + "time": "0:01:22.843981", + "diversity": { + "train": { + "mean_centered_corr": -0.01568707637488842, + "mean_pairwise_disagreement": 0.023222997836599382, + "member_std": 0.024020660668611526 + }, + "val": { + "mean_centered_corr": -0.00023190633510239422, + "mean_pairwise_disagreement": 0.07023343917158961, + "member_std": 0.050259560346603394 + }, + "test": { + "mean_centered_corr": 0.006958201061934233, + "mean_pairwise_disagreement": 0.04134691594369014, + "member_std": 0.04727793484926224 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 20, + 31 + ], + "head_selection_time": "0:00:15.838549" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-best-head-evaluation/0/report.json new file mode 100644 index 000000000..67ae69ad9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-best-head-evaluation/0/report.json @@ -0,0 +1,262 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9509438009438009, + "recall": 0.9745069269279171, + "f1-score": 0.9625811844722908, + "support": 181971.0 + }, + "1": { + "precision": 0.8774048625792812, + "recall": 0.7839854542019883, + "f1-score": 0.8280686877252491, + "support": 42349.0 + }, + "accuracy": 0.9385386947218259, + "macro avg": { + "precision": 0.9141743317615411, + "recall": 0.8792461905649527, + "f1-score": 0.89532493609877, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9370605069851746, + "recall": 0.9385386947218259, + "f1-score": 0.9371867937592895, + "support": 224320.0 + }, + "cross-entropy": 0.14097183098455576, + "roc-auc": 0.9772540623630491, + "score": 0.9772540623630491 + }, + "val": { + "0": { + "precision": 0.9410365039753932, + "recall": 0.9685801326085658, + "f1-score": 0.9546096785588132, + "support": 16741.0 + }, + "1": { + "precision": 0.8190574475404196, + "recall": 0.7009125699146306, + "f1-score": 0.7553934010152284, + "support": 3397.0 + }, + "accuracy": 0.923428344423478, + "macro avg": { + "precision": 0.8800469757579064, + "recall": 0.8347463512615982, + "f1-score": 0.8550015397870208, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9204603367934683, + "recall": 0.923428344423478, + "f1-score": 0.9210046683882125, + "support": 20138.0 + }, + "cross-entropy": 0.181709302619563, + "roc-auc": 0.9554763470552776, + "score": 0.9554763470552776 + }, + "test": { + "0": { + "precision": 0.9398939744642724, + "recall": 0.9574807940975127, + "f1-score": 0.9486058779201205, + "support": 13147.0 + }, + "1": { + "precision": 0.8073742246726395, + "recall": 0.7442820838627701, + "f1-score": 0.7745454545454545, + "support": 3148.0 + }, + "accuracy": 0.9162933415158024, + "macro avg": { + "precision": 0.873634099568456, + "recall": 0.8508814389801413, + "f1-score": 0.8615756662327876, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9142927365174138, + "recall": 0.9162933415158024, + "f1-score": 0.914979476399197, + "support": 16295.0 + }, + "cross-entropy": 0.18955508005758365, + "roc-auc": 0.959416932798502, + "score": 0.959416932798502 + } + }, + "time": "0:02:48.403690", + "diversity": { + "train": { + "mean_centered_corr": 0.05291927233338356, + "mean_pairwise_disagreement": 0.03400679940235148, + "member_std": 0.03158120810985565 + }, + "val": { + "mean_centered_corr": 0.06288693100214005, + "mean_pairwise_disagreement": 0.03244768356405319, + "member_std": 0.0349886491894722 + }, + "test": { + "mean_centered_corr": 0.07999585568904877, + "mean_pairwise_disagreement": 0.041502439893495925, + "member_std": 0.0369756743311882 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31 + ], + "head_selection_time": "0:00:00.916152" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-best-head-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-best-head-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-best-head-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-best-head-evaluation/1/report.json new file mode 100644 index 000000000..1bddbac12 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-best-head-evaluation/1/report.json @@ -0,0 +1,262 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9421781614647566, + "recall": 0.9823927988525645, + "f1-score": 0.9618653301372848, + "support": 181971.0 + }, + "1": { + "precision": 0.9073506448441385, + "recall": 0.7409383928782262, + "f1-score": 0.8157439783702278, + "support": 42349.0 + }, + "accuracy": 0.9368090228245364, + "macro avg": { + "precision": 0.9247644031544475, + "recall": 0.8616655958653954, + "f1-score": 0.8888046542537563, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9356031324822024, + "recall": 0.9368090228245364, + "f1-score": 0.934279322977945, + "support": 224320.0 + }, + "cross-entropy": 0.1448925441874488, + "roc-auc": 0.975466291176613, + "score": 0.975466291176613 + }, + "val": { + "0": { + "precision": 0.9327206716587247, + "recall": 0.9821396571292037, + "f1-score": 0.9567924583199977, + "support": 16741.0 + }, + "1": { + "precision": 0.8808764940239043, + "recall": 0.6508684133058581, + "f1-score": 0.7486033519553073, + "support": 3397.0 + }, + "accuracy": 0.9262588141821432, + "macro avg": { + "precision": 0.9067985828413145, + "recall": 0.8165040352175309, + "f1-score": 0.8526979051376524, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9239752812810563, + "recall": 0.9262588141821432, + "f1-score": 0.9216738569533847, + "support": 20138.0 + }, + "cross-entropy": 0.17975617136163471, + "roc-auc": 0.9563250405399748, + "score": 0.9563250405399748 + }, + "test": { + "0": { + "precision": 0.9304106791159092, + "recall": 0.9701833117821557, + "f1-score": 0.9498808459934466, + "support": 13147.0 + }, + "1": { + "precision": 0.848414539829853, + "recall": 0.6969504447268107, + "f1-score": 0.7652598535054064, + "support": 3148.0 + }, + "accuracy": 0.9173979748389076, + "macro avg": { + "precision": 0.8894126094728811, + "recall": 0.8335668782544832, + "f1-score": 0.8575703497494265, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9145700012102631, + "recall": 0.9173979748389076, + "f1-score": 0.9142142682485954, + "support": 16295.0 + }, + "cross-entropy": 0.1871619815011916, + "roc-auc": 0.9592494275221765, + "score": 0.9592494275221765 + } + }, + "time": "0:02:44.189369", + "diversity": { + "train": { + "mean_centered_corr": 0.05150473117828369, + "mean_pairwise_disagreement": 0.03169338301274677, + "member_std": 0.03007516823709011 + }, + "val": { + "mean_centered_corr": 0.057541172951459885, + "mean_pairwise_disagreement": 0.02847489419777727, + "member_std": 0.031669627875089645 + }, + "test": { + "mean_centered_corr": 0.06821995228528976, + "mean_pairwise_disagreement": 0.035880044343703295, + "member_std": 0.033187758177518845 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21 + ], + "head_selection_time": "0:00:01.162128" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-best-head-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-best-head-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-best-head-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-best-head-evaluation/2/report.json new file mode 100644 index 000000000..28b72c8fb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-best-head-evaluation/2/report.json @@ -0,0 +1,262 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9441356426540986, + "recall": 0.981964159124256, + "f1-score": 0.9626784256787516, + "support": 181971.0 + }, + "1": { + "precision": 0.9063837069998288, + "recall": 0.7503364896455642, + "f1-score": 0.8210110196752232, + "support": 42349.0 + }, + "accuracy": 0.9382355563480742, + "macro avg": { + "precision": 0.9252596748269637, + "recall": 0.8661503243849101, + "f1-score": 0.8918447226769874, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9370085174623071, + "recall": 0.9382355563480742, + "f1-score": 0.9359332715380446, + "support": 224320.0 + }, + "cross-entropy": 0.14035186091185498, + "roc-auc": 0.977574758838111, + "score": 0.977574758838111 + }, + "val": { + "0": { + "precision": 0.9334280841387151, + "recall": 0.980765784600681, + "f1-score": 0.9565116075849815, + "support": 16741.0 + }, + "1": { + "precision": 0.8736263736263736, + "recall": 0.6552840741831027, + "f1-score": 0.7488645920941968, + "support": 3397.0 + }, + "accuracy": 0.9258615552686463, + "macro avg": { + "precision": 0.9035272288825444, + "recall": 0.8180249293918919, + "f1-score": 0.8526880998395892, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9233403688437293, + "recall": 0.9258615552686463, + "f1-score": 0.92148444939538, + "support": 20138.0 + }, + "cross-entropy": 0.1855825324099513, + "roc-auc": 0.9561054927874199, + "score": 0.9561054927874199 + }, + "test": { + "0": { + "precision": 0.9291811342592593, + "recall": 0.9770289799954363, + "f1-score": 0.9525045419153906, + "support": 13147.0 + }, + "1": { + "precision": 0.877782274382841, + "recall": 0.6890088945362135, + "f1-score": 0.7720234917245061, + "support": 3148.0 + }, + "accuracy": 0.9213869285056766, + "macro avg": { + "precision": 0.9034817043210501, + "recall": 0.8330189372658249, + "f1-score": 0.8622640168199484, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9192514864598752, + "recall": 0.9213869285056766, + "f1-score": 0.9176377517342981, + "support": 16295.0 + }, + "cross-entropy": 0.18959449862593228, + "roc-auc": 0.9608390423255206, + "score": 0.9608390423255206 + } + }, + "time": "0:02:48.224028", + "diversity": { + "train": { + "mean_centered_corr": 0.05968523025512695, + "mean_pairwise_disagreement": 0.033705988852330775, + "member_std": 0.031936075538396835 + }, + "val": { + "mean_centered_corr": 0.06932749599218369, + "mean_pairwise_disagreement": 0.031030346255994927, + "member_std": 0.03422032669186592 + }, + "test": { + "mean_centered_corr": 0.08440784364938736, + "mean_pairwise_disagreement": 0.03857926931871047, + "member_std": 0.03628651052713394 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 1 + ], + "head_selection_time": "0:00:01.010209" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-evaluation/0.toml new file mode 100644 index 000000000..d85b237a3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-evaluation/0.toml @@ -0,0 +1,34 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-evaluation/0/report.json new file mode 100644 index 000000000..cd297e4bc --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-evaluation/0/report.json @@ -0,0 +1,258 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9482836172363248, + "recall": 0.984772298882789, + "f1-score": 0.966183574879227, + "support": 181971.0 + }, + "1": { + "precision": 0.9216057939853453, + "recall": 0.769227136414083, + "f1-score": 0.8385502471169687, + "support": 42349.0 + }, + "accuracy": 0.9440798858773182, + "macro avg": { + "precision": 0.9349447056108351, + "recall": 0.8769997176484361, + "f1-score": 0.9023669109980978, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.943247155320955, + "recall": 0.9440798858773182, + "f1-score": 0.9420878910462924, + "support": 224320.0 + }, + "cross-entropy": 0.13271224543526663, + "roc-auc": 0.9813876825616359, + "score": 0.9813876825616359 + }, + "val": { + "0": { + "precision": 0.9351403678606002, + "recall": 0.9809449853652709, + "f1-score": 0.9574951897848522, + "support": 16741.0 + }, + "1": { + "precision": 0.8762126503686457, + "recall": 0.6647041507212246, + "f1-score": 0.7559424171409441, + "support": 3397.0 + }, + "accuracy": 0.9275995630151952, + "macro avg": { + "precision": 0.9056765091146229, + "recall": 0.8228245680432478, + "f1-score": 0.8567188034628981, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9252000830100605, + "recall": 0.9275995630151952, + "f1-score": 0.9234960454472141, + "support": 20138.0 + }, + "cross-entropy": 0.17351272311805546, + "roc-auc": 0.9583121450834431, + "score": 0.9583121450834431 + }, + "test": { + "0": { + "precision": 0.9321849105974706, + "recall": 0.975507720392485, + "f1-score": 0.9533543950938487, + "support": 13147.0 + }, + "1": { + "precision": 0.8730784391013008, + "recall": 0.7036213468869124, + "f1-score": 0.7792436235708003, + "support": 3148.0 + }, + "accuracy": 0.9229825099723842, + "macro avg": { + "precision": 0.9026316748493857, + "recall": 0.8395645336396986, + "f1-score": 0.8662990093323245, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9207662439960627, + "recall": 0.9229825099723842, + "f1-score": 0.9197182669100772, + "support": 16295.0 + }, + "cross-entropy": 0.17696141048282132, + "roc-auc": 0.9626260391126089, + "score": 0.9626260391126089 + } + }, + "time": "0:02:48.403690", + "diversity": { + "train": { + "mean_centered_corr": 0.05291927233338356, + "mean_pairwise_disagreement": 0.03400679940235148, + "member_std": 0.03158120810985565 + }, + "val": { + "mean_centered_corr": 0.06288693100214005, + "mean_pairwise_disagreement": 0.03244768356405319, + "member_std": 0.0349886491894722 + }, + "test": { + "mean_centered_corr": 0.07999585568904877, + "mean_pairwise_disagreement": 0.041502439893495925, + "member_std": 0.0369756743311882 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-evaluation/1.toml new file mode 100644 index 000000000..12aa82432 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-evaluation/1.toml @@ -0,0 +1,34 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 1 +head_selection = true + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-evaluation/1/report.json new file mode 100644 index 000000000..a5a6399d1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-evaluation/1/report.json @@ -0,0 +1,258 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9421632002014817, + "recall": 0.9867781129960268, + "f1-score": 0.9639547024766547, + "support": 181971.0 + }, + "1": { + "precision": 0.9286730700818214, + "recall": 0.7397105008382725, + "f1-score": 0.8234907532761137, + "support": 42349.0 + }, + "accuracy": 0.9401346291012839, + "macro avg": { + "precision": 0.9354181351416515, + "recall": 0.8632443069171496, + "f1-score": 0.8937227278763842, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9396164209555941, + "recall": 0.9401346291012839, + "f1-score": 0.9374367469457449, + "support": 224320.0 + }, + "cross-entropy": 0.1388001827221654, + "roc-auc": 0.978892038769101, + "score": 0.978892038769101 + }, + "val": { + "0": { + "precision": 0.9301735798016231, + "recall": 0.9859028731855922, + "f1-score": 0.9572277801942873, + "support": 16741.0 + }, + "1": { + "precision": 0.9014202172096909, + "recall": 0.6352664115395937, + "f1-score": 0.7452944223795545, + "support": 3397.0 + }, + "accuracy": 0.9267553878240143, + "macro avg": { + "precision": 0.915796898505657, + "recall": 0.810584642362593, + "f1-score": 0.8512611012869209, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9253232881775892, + "recall": 0.9267553878240143, + "f1-score": 0.9214775757799141, + "support": 20138.0 + }, + "cross-entropy": 0.17373390370141575, + "roc-auc": 0.9586997364143323, + "score": 0.9586997364143323 + }, + "test": { + "0": { + "precision": 0.9266714593817398, + "recall": 0.9804518141020765, + "f1-score": 0.952803341094726, + "support": 13147.0 + }, + "1": { + "precision": 0.8922431865828092, + "recall": 0.6759847522236341, + "f1-score": 0.7692029640339779, + "support": 3148.0 + }, + "accuracy": 0.9216324025774778, + "macro avg": { + "precision": 0.9094573229822744, + "recall": 0.8282182831628553, + "f1-score": 0.861003152564352, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9200203269625293, + "recall": 0.9216324025774778, + "f1-score": 0.9173339340994984, + "support": 16295.0 + }, + "cross-entropy": 0.17809500689433352, + "roc-auc": 0.9622916084556132, + "score": 0.9622916084556132 + } + }, + "time": "0:02:44.189369", + "diversity": { + "train": { + "mean_centered_corr": 0.05150473117828369, + "mean_pairwise_disagreement": 0.03169338301274677, + "member_std": 0.03007516823709011 + }, + "val": { + "mean_centered_corr": 0.057541172951459885, + "mean_pairwise_disagreement": 0.02847489419777727, + "member_std": 0.031669627875089645 + }, + "test": { + "mean_centered_corr": 0.06821995228528976, + "mean_pairwise_disagreement": 0.035880044343703295, + "member_std": 0.033187758177518845 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-evaluation/2.toml new file mode 100644 index 000000000..af2af4423 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-evaluation/2.toml @@ -0,0 +1,34 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 2 +head_selection = true + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-evaluation/2/report.json new file mode 100644 index 000000000..7b3d57080 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-evaluation/2/report.json @@ -0,0 +1,258 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9470221716165355, + "recall": 0.9872562111545247, + "f1-score": 0.9667207464626163, + "support": 181971.0 + }, + "1": { + "precision": 0.9330117280027731, + "recall": 0.7626862499704834, + "f1-score": 0.8392947626905037, + "support": 42349.0 + }, + "accuracy": 0.9448600213980028, + "macro avg": { + "precision": 0.9400169498096542, + "recall": 0.874971230562504, + "f1-score": 0.90300775457656, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9443771632508113, + "recall": 0.9448600213980028, + "f1-score": 0.942664206756994, + "support": 224320.0 + }, + "cross-entropy": 0.1304695324130005, + "roc-auc": 0.9818820805222398, + "score": 0.9818820805222398 + }, + "val": { + "0": { + "precision": 0.9318168958297969, + "recall": 0.9836927304223164, + "f1-score": 0.9570523624106468, + "support": 16741.0 + }, + "1": { + "precision": 0.8892494929006085, + "recall": 0.6452752428613483, + "f1-score": 0.7478676219720232, + "support": 3397.0 + }, + "accuracy": 0.926606415731453, + "macro avg": { + "precision": 0.9105331943652026, + "recall": 0.8144839866418323, + "f1-score": 0.852459992191335, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9246363680837221, + "recall": 0.926606415731453, + "f1-score": 0.9217658114487834, + "support": 20138.0 + }, + "cross-entropy": 0.17503044041039684, + "roc-auc": 0.9582475934195426, + "score": 0.9582475934195426 + }, + "test": { + "0": { + "precision": 0.9284580989470648, + "recall": 0.9792348064197155, + "f1-score": 0.9531706955910118, + "support": 13147.0 + }, + "1": { + "precision": 0.8876080691642652, + "recall": 0.6848792884371029, + "f1-score": 0.7731755424063116, + "support": 3148.0 + }, + "accuracy": 0.9223688247928813, + "macro avg": { + "precision": 0.908033084055665, + "recall": 0.8320570474284092, + "f1-score": 0.8631731189986617, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9205663595327503, + "recall": 0.9223688247928813, + "f1-score": 0.9183977749266709, + "support": 16295.0 + }, + "cross-entropy": 0.17851016154157287, + "roc-auc": 0.9625649084455907, + "score": 0.9625649084455907 + } + }, + "time": "0:02:48.224028", + "diversity": { + "train": { + "mean_centered_corr": 0.05968523025512695, + "mean_pairwise_disagreement": 0.033705988852330775, + "member_std": 0.031936075538396835 + }, + "val": { + "mean_centered_corr": 0.06932749599218369, + "mean_pairwise_disagreement": 0.031030346255994927, + "member_std": 0.03422032669186592 + }, + "test": { + "mean_centered_corr": 0.08440784364938736, + "mean_pairwise_disagreement": 0.03857926931871047, + "member_std": 0.03628651052713394 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..0b339388c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/0/report.json @@ -0,0 +1,275 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9453926712602988, + "recall": 0.9862120887394146, + "f1-score": 0.9653710741556594, + "support": 181971.0 + }, + "1": { + "precision": 0.9272584947234142, + "recall": 0.7552244444969185, + "f1-score": 0.8324462201168646, + "support": 42349.0 + }, + "accuracy": 0.9426043152639088, + "macro avg": { + "precision": 0.9363255829918564, + "recall": 0.8707182666181665, + "f1-score": 0.898908647136262, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9419691502092978, + "recall": 0.9426043152639088, + "f1-score": 0.9402764118710262, + "support": 224320.0 + }, + "cross-entropy": 0.13432360872538882, + "roc-auc": 0.9807020207732833, + "score": 0.9807020207732833 + }, + "val": { + "0": { + "precision": 0.9332728098048116, + "recall": 0.9824980586583836, + "f1-score": 0.9572530190600902, + "support": 16741.0 + }, + "1": { + "precision": 0.8834526650755767, + "recall": 0.6538121872240212, + "f1-score": 0.7514802909829132, + "support": 3397.0 + }, + "accuracy": 0.927053332009137, + "macro avg": { + "precision": 0.9083627374401941, + "recall": 0.8181551229412024, + "f1-score": 0.8543666550215017, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9248688455757318, + "recall": 0.927053332009137, + "f1-score": 0.9225420270411127, + "support": 20138.0 + }, + "cross-entropy": 0.17301902153958362, + "roc-auc": 0.9589474980445032, + "score": 0.9589474980445032 + }, + "test": { + "0": { + "precision": 0.9310144506571781, + "recall": 0.9752034684718948, + "f1-score": 0.952596775391931, + "support": 13147.0 + }, + "1": { + "precision": 0.8708399366085579, + "recall": 0.6982210927573063, + "f1-score": 0.7750352609308886, + "support": 3148.0 + }, + "accuracy": 0.921693771095428, + "macro avg": { + "precision": 0.900927193632868, + "recall": 0.8367122806146006, + "f1-score": 0.8638160181614098, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9193894509502093, + "recall": 0.921693771095428, + "f1-score": 0.9182940047553332, + "support": 16295.0 + }, + "cross-entropy": 0.17739591453615167, + "roc-auc": 0.9625170670540112, + "score": 0.9625170670540112 + } + }, + "time": "0:02:48.403690", + "diversity": { + "train": { + "mean_centered_corr": 0.05291927233338356, + "mean_pairwise_disagreement": 0.03400679940235148, + "member_std": 0.03158120810985565 + }, + "val": { + "mean_centered_corr": 0.06288693100214005, + "mean_pairwise_disagreement": 0.03244768356405319, + "member_std": 0.0349886491894722 + }, + "test": { + "mean_centered_corr": 0.07999585568904877, + "mean_pairwise_disagreement": 0.041502439893495925, + "member_std": 0.0369756743311882 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31, + 6, + 9, + 15, + 28, + 3, + 23, + 26, + 2, + 20, + 13, + 6, + 15, + 5 + ], + "head_selection_time": "0:00:13.989049" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/1/report.json new file mode 100644 index 000000000..7a16f3ac1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/1/report.json @@ -0,0 +1,277 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9438133041335889, + "recall": 0.9852284155167582, + "f1-score": 0.9640762837868722, + "support": 181971.0 + }, + "1": { + "precision": 0.921778605517402, + "recall": 0.7479751587994994, + "f1-score": 0.8258313454042991, + "support": 42349.0 + }, + "accuracy": 0.9404377674750357, + "macro avg": { + "precision": 0.9327959548254954, + "recall": 0.8666017871581289, + "f1-score": 0.8949538145955857, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9396534100015593, + "recall": 0.9404377674750357, + "f1-score": 0.9379772516204866, + "support": 224320.0 + }, + "cross-entropy": 0.138611871152865, + "roc-auc": 0.9786723684054657, + "score": 0.9786723684054657 + }, + "val": { + "0": { + "precision": 0.9314650545228544, + "recall": 0.984767935009856, + "f1-score": 0.9573751451800232, + "support": 16741.0 + }, + "1": { + "precision": 0.8954489544895449, + "recall": 0.6429202237268178, + "f1-score": 0.748457847840987, + "support": 3397.0 + }, + "accuracy": 0.927102989373324, + "macro avg": { + "precision": 0.9134570045061996, + "recall": 0.8138440793683369, + "f1-score": 0.8529164965105052, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9253896402904007, + "recall": 0.927102989373324, + "f1-score": 0.9221337081425465, + "support": 20138.0 + }, + "cross-entropy": 0.17287177515474142, + "roc-auc": 0.9593694489371631, + "score": 0.9593694489371631 + }, + "test": { + "0": { + "precision": 0.9287417792874177, + "recall": 0.9774853578763216, + "f1-score": 0.9524903646605396, + "support": 13147.0 + }, + "1": { + "precision": 0.8795768917819365, + "recall": 0.6867852604828463, + "f1-score": 0.7713164466642882, + "support": 3148.0 + }, + "accuracy": 0.9213255599877263, + "macro avg": { + "precision": 0.9041593355346771, + "recall": 0.832135309179584, + "f1-score": 0.861903405662414, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9192437083535575, + "recall": 0.9213255599877263, + "f1-score": 0.9174897206683826, + "support": 16295.0 + }, + "cross-entropy": 0.17752804180302853, + "roc-auc": 0.9625145058481993, + "score": 0.9625145058481993 + } + }, + "time": "0:02:44.189369", + "diversity": { + "train": { + "mean_centered_corr": 0.05150473117828369, + "mean_pairwise_disagreement": 0.03169338301274677, + "member_std": 0.03007516823709011 + }, + "val": { + "mean_centered_corr": 0.057541172951459885, + "mean_pairwise_disagreement": 0.02847489419777727, + "member_std": 0.031669627875089645 + }, + "test": { + "mean_centered_corr": 0.06821995228528976, + "mean_pairwise_disagreement": 0.035880044343703295, + "member_std": 0.033187758177518845 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21, + 14, + 26, + 8, + 24, + 18, + 6, + 17, + 9, + 30, + 14, + 5, + 10, + 15, + 26, + 28 + ], + "head_selection_time": "0:00:15.746127" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/2/report.json new file mode 100644 index 000000000..bd1780ddf --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/2/report.json @@ -0,0 +1,269 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9458610786757976, + "recall": 0.9882288936149166, + "f1-score": 0.9665809354113085, + "support": 181971.0 + }, + "1": { + "precision": 0.9373647581729926, + "recall": 0.7569482160145458, + "f1-score": 0.8375507857917358, + "support": 42349.0 + }, + "accuracy": 0.9445657988587732, + "macro avg": { + "precision": 0.9416129184243951, + "recall": 0.8725885548147312, + "f1-score": 0.9020658606015222, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9442570724482062, + "recall": 0.9445657988587732, + "f1-score": 0.942221547901326, + "support": 224320.0 + }, + "cross-entropy": 0.1303272598008576, + "roc-auc": 0.9816630888898851, + "score": 0.9816630888898851 + }, + "val": { + "0": { + "precision": 0.9299154929577464, + "recall": 0.9859626067737889, + "f1-score": 0.9571192484996086, + "support": 16741.0 + }, + "1": { + "precision": 0.9015912897822446, + "recall": 0.6337945245805122, + "f1-score": 0.7443388072601556, + "support": 3397.0 + }, + "accuracy": 0.9265567583672659, + "macro avg": { + "precision": 0.9157533913699956, + "recall": 0.8098785656771506, + "f1-score": 0.850729027879882, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9251375945474187, + "recall": 0.9265567583672659, + "f1-score": 0.921226152914624, + "support": 20138.0 + }, + "cross-entropy": 0.1753619698028255, + "roc-auc": 0.9591019138539669, + "score": 0.9591019138539669 + }, + "test": { + "0": { + "precision": 0.9280829313944281, + "recall": 0.9806039400623716, + "f1-score": 0.9536208299430431, + "support": 13147.0 + }, + "1": { + "precision": 0.8939267886855241, + "recall": 0.6826556543837357, + "f1-score": 0.774135446685879, + "support": 3148.0 + }, + "accuracy": 0.9230438784903344, + "macro avg": { + "precision": 0.911004860039976, + "recall": 0.8316297972230536, + "f1-score": 0.863878138314461, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9214843712687681, + "recall": 0.9230438784903344, + "f1-score": 0.9189463907596401, + "support": 16295.0 + }, + "cross-entropy": 0.17871078362700502, + "roc-auc": 0.9629950822915426, + "score": 0.9629950822915426 + } + }, + "time": "0:02:48.224028", + "diversity": { + "train": { + "mean_centered_corr": 0.05968523025512695, + "mean_pairwise_disagreement": 0.033705988852330775, + "member_std": 0.031936075538396835 + }, + "val": { + "mean_centered_corr": 0.06932749599218369, + "mean_pairwise_disagreement": 0.031030346255994927, + "member_std": 0.03422032669186592 + }, + "test": { + "mean_centered_corr": 0.08440784364938736, + "mean_pairwise_disagreement": 0.03857926931871047, + "member_std": 0.03628651052713394 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 1, + 6, + 25, + 11, + 5, + 18, + 4, + 14 + ], + "head_selection_time": "0:00:09.905004" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..a89b4d488 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-best-head-evaluation/0/report.json @@ -0,0 +1,621 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2420, + "metrics": { + "train": { + "0": { + "precision": 0.9511390416339356, + "recall": 0.9648020838485253, + "f1-score": 0.9579218455024607, + "support": 181971.0 + }, + "1": { + "precision": 0.8388070970177426, + "recall": 0.7870315709934119, + "f1-score": 0.8120949271477998, + "support": 42349.0 + }, + "accuracy": 0.9312410841654779, + "macro avg": { + "precision": 0.8949730693258391, + "recall": 0.8759168274209685, + "f1-score": 0.8850083863251303, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9299320804956013, + "recall": 0.9312410841654779, + "f1-score": 0.9303914239377251, + "support": 224320.0 + }, + "cross-entropy": 0.15615645016500299, + "roc-auc": 0.9724991426058968, + "score": 0.9724991426058968 + }, + "val": { + "0": { + "precision": 0.9410138248847926, + "recall": 0.9758078967803596, + "f1-score": 0.958095070525791, + "support": 16741.0 + }, + "1": { + "precision": 0.8542116630669546, + "recall": 0.6985575507801001, + "f1-score": 0.768582995951417, + "support": 3397.0 + }, + "accuracy": 0.9290396265766213, + "macro avg": { + "precision": 0.8976127439758737, + "recall": 0.8371827237802298, + "f1-score": 0.863339033238604, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9263715096749804, + "recall": 0.9290396265766213, + "f1-score": 0.9261270241791256, + "support": 20138.0 + }, + "cross-entropy": 0.17528768630111824, + "roc-auc": 0.957813544584969, + "score": 0.957813544584969 + }, + "test": { + "0": { + "precision": 0.9342733348020856, + "recall": 0.9676732334372861, + "f1-score": 0.9506800179345389, + "support": 13147.0 + }, + "1": { + "precision": 0.8412994772218073, + "recall": 0.71569250317662, + "f1-score": 0.7734294541709578, + "support": 3148.0 + }, + "accuracy": 0.9189935563056152, + "macro avg": { + "precision": 0.8877864060119465, + "recall": 0.841682868306953, + "f1-score": 0.8620547360527484, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9163118924171383, + "recall": 0.9189935563056152, + "f1-score": 0.9164373192706695, + "support": 16295.0 + }, + "cross-entropy": 0.1803789882797406, + "roc-auc": 0.961490313471295, + "score": 0.961490313471295 + } + }, + "time": "0:03:46.650441", + "diversity": { + "train": { + "mean_centered_corr": 0.0506281740963459, + "mean_pairwise_disagreement": 0.03327413710540242, + "member_std": 0.03079444356262684 + }, + "val": { + "mean_centered_corr": 0.056434109807014465, + "mean_pairwise_disagreement": 0.030276074921749604, + "member_std": 0.03229342773556709 + }, + "test": { + "mean_centered_corr": 0.06734751909971237, + "mean_pairwise_disagreement": 0.037917825574834946, + "member_std": 0.03452758118510246 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25 + ], + "head_selection_time": "0:00:00.860463" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-best-head-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-best-head-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-best-head-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-best-head-evaluation/1/report.json new file mode 100644 index 000000000..008857947 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-best-head-evaluation/1/report.json @@ -0,0 +1,621 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9352279269377715, + "recall": 0.9819916360299169, + "f1-score": 0.9580394648309435, + "support": 181971.0 + }, + "1": { + "precision": 0.9014436090225564, + "recall": 0.7077616944910151, + "f1-score": 0.7929469966533949, + "support": 42349.0 + }, + "accuracy": 0.9302202211126962, + "macro avg": { + "precision": 0.9183357679801639, + "recall": 0.844876665260466, + "f1-score": 0.8754932307421692, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9288498417051063, + "recall": 0.9302202211126962, + "f1-score": 0.9268719321327845, + "support": 224320.0 + }, + "cross-entropy": 0.16027100539450811, + "roc-auc": 0.9698985230711175, + "score": 0.9698985230711175 + }, + "val": { + "0": { + "precision": 0.9274849862490879, + "recall": 0.9870975449495251, + "f1-score": 0.9563632154638578, + "support": 16741.0 + }, + "1": { + "precision": 0.9069366652305041, + "recall": 0.6196644097733294, + "f1-score": 0.7362714235746765, + "support": 3397.0 + }, + "accuracy": 0.9251166948058397, + "macro avg": { + "precision": 0.917210825739796, + "recall": 0.8033809773614272, + "f1-score": 0.8463173195192671, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9240187708106069, + "recall": 0.9251166948058397, + "f1-score": 0.9192367968995738, + "support": 20138.0 + }, + "cross-entropy": 0.18037389209877025, + "roc-auc": 0.9580043948939159, + "score": 0.9580043948939159 + }, + "test": { + "0": { + "precision": 0.9213914775556662, + "recall": 0.9851677188712253, + "f1-score": 0.9522129098661961, + "support": 13147.0 + }, + "1": { + "precision": 0.9128686327077749, + "recall": 0.6489834815756036, + "f1-score": 0.7586334942443371, + "support": 3148.0 + }, + "accuracy": 0.9202209266646211, + "macro avg": { + "precision": 0.9171300551317205, + "recall": 0.8170756002234145, + "f1-score": 0.8554232020552666, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9197449653997188, + "recall": 0.9202209266646211, + "f1-score": 0.9148156714263305, + "support": 16295.0 + }, + "cross-entropy": 0.18507285757654685, + "roc-auc": 0.9620360194454477, + "score": 0.9620360194454477 + } + }, + "time": "0:03:27.053329", + "diversity": { + "train": { + "mean_centered_corr": 0.047042831778526306, + "mean_pairwise_disagreement": 0.027546063946666054, + "member_std": 0.02635623700916767 + }, + "val": { + "mean_centered_corr": 0.04908687248826027, + "mean_pairwise_disagreement": 0.024034264382214333, + "member_std": 0.02737056463956833 + }, + "test": { + "mean_centered_corr": 0.05689273402094841, + "mean_pairwise_disagreement": 0.0295994219481535, + "member_std": 0.02851640060544014 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 24 + ], + "head_selection_time": "0:00:01.177794" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-best-head-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-best-head-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-best-head-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-best-head-evaluation/2/report.json new file mode 100644 index 000000000..e420fe840 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-best-head-evaluation/2/report.json @@ -0,0 +1,621 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9298072451036122, + "recall": 0.9887674409658682, + "f1-score": 0.9583813828129786, + "support": 181971.0 + }, + "1": { + "precision": 0.9336579032781565, + "recall": 0.6792604311790125, + "f1-score": 0.7863967522792821, + "support": 42349.0 + }, + "accuracy": 0.9303361269614836, + "macro avg": { + "precision": 0.9317325741908844, + "recall": 0.8340139360724403, + "f1-score": 0.8723890675461303, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9305342044609312, + "recall": 0.9303361269614836, + "f1-score": 0.9259126902377668, + "support": 224320.0 + }, + "cross-entropy": 0.15872104617363245, + "roc-auc": 0.9706236144688893, + "score": 0.9706236144688893 + }, + "val": { + "0": { + "precision": 0.9251312995865459, + "recall": 0.9890687533600143, + "f1-score": 0.9560322180201507, + "support": 16741.0 + }, + "1": { + "precision": 0.9183035714285714, + "recall": 0.6055342949661466, + "f1-score": 0.7298208266808586, + "support": 3397.0 + }, + "accuracy": 0.924371834343033, + "macro avg": { + "precision": 0.9217174355075587, + "recall": 0.7973015241630805, + "f1-score": 0.8429265223505047, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9239795569828794, + "recall": 0.924371834343033, + "f1-score": 0.917873508298253, + "support": 20138.0 + }, + "cross-entropy": 0.18042645510918942, + "roc-auc": 0.9575815999587967, + "score": 0.9575815999587967 + }, + "test": { + "0": { + "precision": 0.9182661661590764, + "recall": 0.9861565376131437, + "f1-score": 0.9510012469742537, + "support": 13147.0 + }, + "1": { + "precision": 0.9163602941176471, + "recall": 0.633418043202033, + "f1-score": 0.749060856498873, + "support": 3148.0 + }, + "accuracy": 0.9180116600184105, + "macro avg": { + "precision": 0.9173132301383617, + "recall": 0.8097872904075883, + "f1-score": 0.8500310517365633, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9178979743710175, + "recall": 0.9180116600184105, + "f1-score": 0.9119887677342108, + "support": 16295.0 + }, + "cross-entropy": 0.18508615173756227, + "roc-auc": 0.9611081622343147, + "score": 0.9611081622343147 + } + }, + "time": "0:03:33.015302", + "diversity": { + "train": { + "mean_centered_corr": 0.02876308374106884, + "mean_pairwise_disagreement": 0.026535222219985274, + "member_std": 0.025292295962572098 + }, + "val": { + "mean_centered_corr": 0.041079021990299225, + "mean_pairwise_disagreement": 0.02388218870439131, + "member_std": 0.026205172762274742 + }, + "test": { + "mean_centered_corr": 0.053323205560445786, + "mean_pairwise_disagreement": 0.030486048560314366, + "member_std": 0.028108706697821617 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 27 + ], + "head_selection_time": "0:00:01.182248" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/0.toml new file mode 100644 index 000000000..e655e5952 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/0.toml @@ -0,0 +1,45 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "homesite-insurance" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/0/report.json new file mode 100644 index 000000000..5caf5c656 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/0/report.json @@ -0,0 +1,617 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2420, + "metrics": { + "train": { + "0": { + "precision": 0.9480090078156047, + "recall": 0.9831951244978595, + "f1-score": 0.965281524483674, + "support": 181971.0 + }, + "1": { + "precision": 0.9140890574518893, + "recall": 0.7683062173841176, + "f1-score": 0.8348814533511238, + "support": 42349.0 + }, + "accuracy": 0.942626604850214, + "macro avg": { + "precision": 0.931049032633747, + "recall": 0.8757506709409886, + "f1-score": 0.9000814889173989, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9416053167583962, + "recall": 0.942626604850214, + "f1-score": 0.9406635117679449, + "support": 224320.0 + }, + "cross-entropy": 0.13436786219631347, + "roc-auc": 0.9804692077039372, + "score": 0.9804692077039372 + }, + "val": { + "0": { + "precision": 0.9353075170842825, + "recall": 0.9810644525416642, + "f1-score": 0.957639718958631, + "support": 16741.0 + }, + "1": { + "precision": 0.8770364623739333, + "recall": 0.6655872828966736, + "f1-score": 0.7568200836820084, + "support": 3397.0 + }, + "accuracy": 0.9278478498361307, + "macro avg": { + "precision": 0.9061719897291078, + "recall": 0.8233258677191688, + "f1-score": 0.8572299013203197, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9254780020951546, + "recall": 0.9278478498361307, + "f1-score": 0.9237642446794232, + "support": 20138.0 + }, + "cross-entropy": 0.1724175473381701, + "roc-auc": 0.9589599740471012, + "score": 0.9589599740471012 + }, + "test": { + "0": { + "precision": 0.931104503589818, + "recall": 0.9765726021145509, + "f1-score": 0.9532967032967034, + "support": 13147.0 + }, + "1": { + "precision": 0.8770949720670391, + "recall": 0.6982210927573063, + "f1-score": 0.7775026529890343, + "support": 3148.0 + }, + "accuracy": 0.9227984044185333, + "macro avg": { + "precision": 0.9040997378284286, + "recall": 0.8373968474359286, + "f1-score": 0.8653996781428688, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9206705051096887, + "recall": 0.9227984044185333, + "f1-score": 0.919335385692006, + "support": 16295.0 + }, + "cross-entropy": 0.17668487055650292, + "roc-auc": 0.9628908822909436, + "score": 0.9628908822909436 + } + }, + "time": "0:03:46.650441", + "diversity": { + "train": { + "mean_centered_corr": 0.0506281740963459, + "mean_pairwise_disagreement": 0.03327413710540242, + "member_std": 0.03079444356262684 + }, + "val": { + "mean_centered_corr": 0.056434109807014465, + "mean_pairwise_disagreement": 0.030276074921749604, + "member_std": 0.03229342773556709 + }, + "test": { + "mean_centered_corr": 0.06734751909971237, + "mean_pairwise_disagreement": 0.037917825574834946, + "member_std": 0.03452758118510246 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/1.toml new file mode 100644 index 000000000..138dbeb9f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/1.toml @@ -0,0 +1,45 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 1 +head_selection = true + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "homesite-insurance" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/1/report.json new file mode 100644 index 000000000..9594849bb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/1/report.json @@ -0,0 +1,617 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9375189502849077, + "recall": 0.9855306614790269, + "f1-score": 0.9609254650231339, + "support": 181971.0 + }, + "1": { + "precision": 0.9202845897668785, + "recall": 0.71777373727833, + "f1-score": 0.8065110972552038, + "support": 42349.0 + }, + "accuracy": 0.9349812767475035, + "macro avg": { + "precision": 0.9289017700258931, + "recall": 0.8516521993786785, + "f1-score": 0.8837182811391688, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9342653040046919, + "recall": 0.9349812767475035, + "f1-score": 0.9317738331552484, + "support": 224320.0 + }, + "cross-entropy": 0.14907449987993657, + "roc-auc": 0.9745732258873414, + "score": 0.9745732258873414 + }, + "val": { + "0": { + "precision": 0.9290293555280621, + "recall": 0.9867988770085419, + "f1-score": 0.9570431306665121, + "support": 16741.0 + }, + "1": { + "precision": 0.9061969439728353, + "recall": 0.6284957315278187, + "f1-score": 0.7422214496784286, + "support": 3397.0 + }, + "accuracy": 0.9263581289105174, + "macro avg": { + "precision": 0.9176131497504487, + "recall": 0.8076473042681802, + "f1-score": 0.8496322901724704, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9251778458422389, + "recall": 0.9263581289105174, + "f1-score": 0.9208057063782751, + "support": 20138.0 + }, + "cross-entropy": 0.17400536493244298, + "roc-auc": 0.9589376508824807, + "score": 0.9589376508824807 + }, + "test": { + "0": { + "precision": 0.9259578131726216, + "recall": 0.9816688217844375, + "f1-score": 0.9529998153959757, + "support": 13147.0 + }, + "1": { + "precision": 0.8977513788714467, + "recall": 0.6721728081321474, + "f1-score": 0.768755676657584, + "support": 3148.0 + }, + "accuracy": 0.9218778766492789, + "macro avg": { + "precision": 0.9118545960220341, + "recall": 0.8269208149582925, + "f1-score": 0.8608777460267798, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9205086658771261, + "recall": 0.9218778766492789, + "f1-score": 0.9174060413089272, + "support": 16295.0 + }, + "cross-entropy": 0.17781928920731452, + "roc-auc": 0.9627469836002609, + "score": 0.9627469836002609 + } + }, + "time": "0:03:27.053329", + "diversity": { + "train": { + "mean_centered_corr": 0.047042831778526306, + "mean_pairwise_disagreement": 0.027546063946666054, + "member_std": 0.02635623700916767 + }, + "val": { + "mean_centered_corr": 0.04908687248826027, + "mean_pairwise_disagreement": 0.024034264382214333, + "member_std": 0.02737056463956833 + }, + "test": { + "mean_centered_corr": 0.05689273402094841, + "mean_pairwise_disagreement": 0.0295994219481535, + "member_std": 0.02851640060544014 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/2.toml new file mode 100644 index 000000000..7cd9bf24e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/2.toml @@ -0,0 +1,45 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 2 +head_selection = true + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "homesite-insurance" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/2/report.json new file mode 100644 index 000000000..4a6d2f99f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/2/report.json @@ -0,0 +1,617 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.938750961020486, + "recall": 0.9863769501733792, + "f1-score": 0.9619748428347098, + "support": 181971.0 + }, + "1": { + "precision": 0.9251441857656189, + "recall": 0.7234645446173463, + "f1-score": 0.8119683036069223, + "support": 42349.0 + }, + "accuracy": 0.9367421540656206, + "macro avg": { + "precision": 0.9319475733930525, + "recall": 0.8549207473953628, + "f1-score": 0.8869715732208161, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9361821605333767, + "recall": 0.9367421540656206, + "f1-score": 0.9336553575914966, + "support": 224320.0 + }, + "cross-entropy": 0.14475981504575822, + "roc-auc": 0.9761314634035192, + "score": 0.9761314634035192 + }, + "val": { + "0": { + "precision": 0.9294680551646496, + "recall": 0.9863210083029688, + "f1-score": 0.9570509476612763, + "support": 16741.0 + }, + "1": { + "precision": 0.9034976822587442, + "recall": 0.6311451280541654, + "f1-score": 0.7431542461005199, + "support": 3397.0 + }, + "accuracy": 0.9264077862747045, + "macro avg": { + "precision": 0.9164828687116968, + "recall": 0.8087330681785672, + "f1-score": 0.8501025968808982, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9250872151228698, + "recall": 0.9264077862747045, + "f1-score": 0.9209695545138987, + "support": 20138.0 + }, + "cross-entropy": 0.17454952390147907, + "roc-auc": 0.9589488871977169, + "score": 0.9589488871977169 + }, + "test": { + "0": { + "precision": 0.9246472315736695, + "recall": 0.9818970107248802, + "f1-score": 0.9524125719344843, + "support": 13147.0 + }, + "1": { + "precision": 0.8980291345329906, + "recall": 0.6658195679796697, + "f1-score": 0.7646844217438891, + "support": 3148.0 + }, + "accuracy": 0.920834611844124, + "macro avg": { + "precision": 0.91133818305333, + "recall": 0.823858289352275, + "f1-score": 0.8585484968391867, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9195049321269646, + "recall": 0.920834611844124, + "f1-score": 0.9161457283137421, + "support": 16295.0 + }, + "cross-entropy": 0.1789803337967533, + "roc-auc": 0.9624131449200803, + "score": 0.9624131449200803 + } + }, + "time": "0:03:33.015302", + "diversity": { + "train": { + "mean_centered_corr": 0.02876308374106884, + "mean_pairwise_disagreement": 0.026535222219985274, + "member_std": 0.025292295962572098 + }, + "val": { + "mean_centered_corr": 0.041079021990299225, + "mean_pairwise_disagreement": 0.02388218870439131, + "member_std": 0.026205172762274742 + }, + "test": { + "mean_centered_corr": 0.053323205560445786, + "mean_pairwise_disagreement": 0.030486048560314366, + "member_std": 0.028108706697821617 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..a6950ee69 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,639 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2420, + "metrics": { + "train": { + "0": { + "precision": 0.9470743368373016, + "recall": 0.9808101290865028, + "f1-score": 0.9636470639051465, + "support": 181971.0 + }, + "1": { + "precision": 0.9026403100342933, + "recall": 0.7644808614134927, + "f1-score": 0.8278357369336197, + "support": 42349.0 + }, + "accuracy": 0.9399696861626248, + "macro avg": { + "precision": 0.9248573234357975, + "recall": 0.8726454952499978, + "f1-score": 0.8957414004193831, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9386857107625842, + "recall": 0.9399696861626248, + "f1-score": 0.9380074691926055, + "support": 224320.0 + }, + "cross-entropy": 0.13890143673129085, + "roc-auc": 0.9784781707405066, + "score": 0.9784781707405066 + }, + "val": { + "0": { + "precision": 0.9363594890510949, + "recall": 0.9808255181888776, + "f1-score": 0.9580768445313184, + "support": 16741.0 + }, + "1": { + "precision": 0.8766333589546502, + "recall": 0.6714748307329997, + "f1-score": 0.7604600766794466, + "support": 3397.0 + }, + "accuracy": 0.9286423676631245, + "macro avg": { + "precision": 0.9064964240028726, + "recall": 0.8261501744609387, + "f1-score": 0.8592684606053824, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9262845231191442, + "recall": 0.9286423676631245, + "f1-score": 0.9247416493583713, + "support": 20138.0 + }, + "cross-entropy": 0.17179269209939912, + "roc-auc": 0.9594844321379927, + "score": 0.9594844321379927 + }, + "test": { + "0": { + "precision": 0.9321799811005307, + "recall": 0.9754316574123374, + "f1-score": 0.9533154921201308, + "support": 13147.0 + }, + "1": { + "precision": 0.8727344365642238, + "recall": 0.7036213468869124, + "f1-score": 0.7791065775589167, + "support": 3148.0 + }, + "accuracy": 0.9229211414544338, + "macro avg": { + "precision": 0.9024572088323772, + "recall": 0.8395265021496249, + "f1-score": 0.8662110348395238, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9206958096245998, + "recall": 0.9229211414544338, + "f1-score": 0.9196604038698268, + "support": 16295.0 + }, + "cross-entropy": 0.1758390088286691, + "roc-auc": 0.9634382071404677, + "score": 0.9634382071404677 + } + }, + "time": "0:03:46.650441", + "diversity": { + "train": { + "mean_centered_corr": 0.0506281740963459, + "mean_pairwise_disagreement": 0.03327413710540242, + "member_std": 0.03079444356262684 + }, + "val": { + "mean_centered_corr": 0.056434109807014465, + "mean_pairwise_disagreement": 0.030276074921749604, + "member_std": 0.03229342773556709 + }, + "test": { + "mean_centered_corr": 0.06734751909971237, + "mean_pairwise_disagreement": 0.037917825574834946, + "member_std": 0.03452758118510246 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25, + 2, + 0, + 28, + 7, + 26, + 18, + 29, + 24, + 15, + 4, + 7, + 27, + 19, + 31, + 0, + 2, + 28, + 11 + ], + "head_selection_time": "0:00:17.095036" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/1/report.json new file mode 100644 index 000000000..83cdf249d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/1/report.json @@ -0,0 +1,631 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9325960268331218, + "recall": 0.989349951365877, + "f1-score": 0.9601350342385392, + "support": 181971.0 + }, + "1": { + "precision": 0.9380335731414868, + "recall": 0.6927436303100427, + "f1-score": 0.7969412148212539, + "support": 42349.0 + }, + "accuracy": 0.9333541369472182, + "macro avg": { + "precision": 0.9353147999873044, + "recall": 0.8410467908379599, + "f1-score": 0.8785381245298965, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9336225721639569, + "recall": 0.9333541369472182, + "f1-score": 0.929325944284444, + "support": 224320.0 + }, + "cross-entropy": 0.15036636948759663, + "roc-auc": 0.9745291516953071, + "score": 0.9745291516953071 + }, + "val": { + "0": { + "precision": 0.9258700631249651, + "recall": 0.9900244907711606, + "f1-score": 0.9568731597482825, + "support": 16741.0 + }, + "1": { + "precision": 0.9253464461332142, + "recall": 0.6093612010597587, + "f1-score": 0.7348242811501597, + "support": 3397.0 + }, + "accuracy": 0.9258118979044593, + "macro avg": { + "precision": 0.9256082546290896, + "recall": 0.7996928459154596, + "f1-score": 0.8458487204492211, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9257817362344607, + "recall": 0.9258118979044593, + "f1-score": 0.9194166079259654, + "support": 20138.0 + }, + "cross-entropy": 0.17630521274856525, + "roc-auc": 0.9596519921503349, + "score": 0.9596519921503349 + }, + "test": { + "0": { + "precision": 0.921048891415577, + "recall": 0.9858522856925535, + "f1-score": 0.9523494617730262, + "support": 13147.0 + }, + "1": { + "precision": 0.9163292847503374, + "recall": 0.6470775095298602, + "f1-score": 0.7585179668590579, + "support": 3148.0 + }, + "accuracy": 0.9204050322184719, + "macro avg": { + "precision": 0.9186890880829572, + "recall": 0.8164648976112068, + "f1-score": 0.855433714316042, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9201371195970944, + "recall": 0.9204050322184719, + "f1-score": 0.914903524615053, + "support": 16295.0 + }, + "cross-entropy": 0.18045087821080671, + "roc-auc": 0.9627588835423583, + "score": 0.9627588835423583 + } + }, + "time": "0:03:27.053329", + "diversity": { + "train": { + "mean_centered_corr": 0.047042831778526306, + "mean_pairwise_disagreement": 0.027546063946666054, + "member_std": 0.02635623700916767 + }, + "val": { + "mean_centered_corr": 0.04908687248826027, + "mean_pairwise_disagreement": 0.024034264382214333, + "member_std": 0.02737056463956833 + }, + "test": { + "mean_centered_corr": 0.05689273402094841, + "mean_pairwise_disagreement": 0.0295994219481535, + "member_std": 0.02851640060544014 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 24, + 21, + 7, + 15, + 26, + 6, + 18, + 12, + 4, + 6, + 8 + ], + "head_selection_time": "0:00:18.810547" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/2/report.json new file mode 100644 index 000000000..f829313d1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/2/report.json @@ -0,0 +1,626 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9369411248537522, + "recall": 0.985777953629974, + "f1-score": 0.9607393145112699, + "support": 181971.0 + }, + "1": { + "precision": 0.9212512171372931, + "recall": 0.7149165269545916, + "f1-score": 0.8050735910015556, + "support": 42349.0 + }, + "accuracy": 0.9346424750356633, + "macro avg": { + "precision": 0.9290961709955227, + "recall": 0.8503472402922828, + "f1-score": 0.8829064527564128, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9339790532512009, + "recall": 0.9346424750356633, + "f1-score": 0.9313514412725357, + "support": 224320.0 + }, + "cross-entropy": 0.14923608977800548, + "roc-auc": 0.9741342548580761, + "score": 0.9741342548580761 + }, + "val": { + "0": { + "precision": 0.929322462749508, + "recall": 0.9872767457141151, + "f1-score": 0.9574233910676012, + "support": 16741.0 + }, + "1": { + "precision": 0.9094772630684232, + "recall": 0.6299676184869002, + "f1-score": 0.7443478260869565, + "support": 3397.0 + }, + "accuracy": 0.9270036746449498, + "macro avg": { + "precision": 0.9193998629089657, + "recall": 0.8086221821005076, + "f1-score": 0.8508856085772789, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9259748540834715, + "recall": 0.9270036746449498, + "f1-score": 0.9214805122196894, + "support": 20138.0 + }, + "cross-entropy": 0.174415932178035, + "roc-auc": 0.9594475404488445, + "score": 0.9594475404488445 + }, + "test": { + "0": { + "precision": 0.9254373386865501, + "recall": 0.9818209477447326, + "f1-score": 0.9527957187673003, + "support": 13147.0 + }, + "1": { + "precision": 0.898167873881551, + "recall": 0.6696315120711563, + "f1-score": 0.7672429481346679, + "support": 3148.0 + }, + "accuracy": 0.9215096655415772, + "macro avg": { + "precision": 0.9118026062840505, + "recall": 0.8257262299079444, + "f1-score": 0.860019333450984, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9201692027426325, + "recall": 0.9215096655415772, + "f1-score": 0.9169491325781916, + "support": 16295.0 + }, + "cross-entropy": 0.17907506203888027, + "roc-auc": 0.9625258379758007, + "score": 0.9625258379758007 + } + }, + "time": "0:03:33.015302", + "diversity": { + "train": { + "mean_centered_corr": 0.02876308374106884, + "mean_pairwise_disagreement": 0.026535222219985274, + "member_std": 0.025292295962572098 + }, + "val": { + "mean_centered_corr": 0.041079021990299225, + "mean_pairwise_disagreement": 0.02388218870439131, + "member_std": 0.026205172762274742 + }, + "test": { + "mean_centered_corr": 0.053323205560445786, + "mean_pairwise_disagreement": 0.030486048560314366, + "member_std": 0.028108706697821617 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 27, + 5, + 16, + 28, + 4, + 6 + ], + "head_selection_time": "0:00:10.772881" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-best-head-evaluation/0/report.json new file mode 100644 index 000000000..4003e9150 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-best-head-evaluation/0/report.json @@ -0,0 +1,657 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.901321291923523, + "min_features_per_member": 256.0, + "mean_features_per_member": 268.59375, + "max_features_per_member": 279.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.842281341552734, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8197222809700668, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9432566752157673, + "recall": 0.9741662132977232, + "f1-score": 0.9584623078898636, + "support": 181971.0 + }, + "1": { + "precision": 0.8708019567965701, + "recall": 0.7481876785756453, + "f1-score": 0.8048517177875151, + "support": 42349.0 + }, + "accuracy": 0.9315041012838802, + "macro avg": { + "precision": 0.9070293160061687, + "recall": 0.8611769459366843, + "f1-score": 0.8816570128386894, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9295780693387408, + "recall": 0.9315041012838802, + "f1-score": 0.9294624198716558, + "support": 224320.0 + }, + "cross-entropy": 0.15818911070182506, + "roc-auc": 0.9710347146934778, + "score": 0.9710347146934778 + }, + "val": { + "0": { + "precision": 0.9363334478415206, + "recall": 0.9768831013678991, + "f1-score": 0.9561785599438711, + "support": 16741.0 + }, + "1": { + "precision": 0.8551646706586826, + "recall": 0.672652340300265, + "f1-score": 0.753007085187016, + "support": 3397.0 + }, + "accuracy": 0.9255636110835237, + "macro avg": { + "precision": 0.8957490592501016, + "recall": 0.824767720834082, + "f1-score": 0.8545928225654436, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9226414061248607, + "recall": 0.9255636110835237, + "f1-score": 0.9219063631145417, + "support": 20138.0 + }, + "cross-entropy": 0.1787492077720551, + "roc-auc": 0.9576967765860231, + "score": 0.9576967765860231 + }, + "test": { + "0": { + "precision": 0.9309967939376275, + "recall": 0.971856697345402, + "f1-score": 0.9509880540359495, + "support": 13147.0 + }, + "1": { + "precision": 0.8560871256320498, + "recall": 0.6991740787801779, + "f1-score": 0.7697149851372618, + "support": 3148.0 + }, + "accuracy": 0.9191776618594661, + "macro avg": { + "precision": 0.8935419597848386, + "recall": 0.8355153880627899, + "f1-score": 0.8603515195866056, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9165251378574827, + "recall": 0.9191776618594661, + "f1-score": 0.915968255269882, + "support": 16295.0 + }, + "cross-entropy": 0.18291214189215663, + "roc-auc": 0.9617315500639867, + "score": 0.9617315500639867 + } + }, + "time": "0:03:52.600341", + "diversity": { + "train": { + "mean_centered_corr": 0.0775546282529831, + "mean_pairwise_disagreement": 0.0356185432101606, + "member_std": 0.03550577908754349 + }, + "val": { + "mean_centered_corr": 0.08212897181510925, + "mean_pairwise_disagreement": 0.03171593825186856, + "member_std": 0.03616226091980934 + }, + "test": { + "mean_centered_corr": 0.10865671932697296, + "mean_pairwise_disagreement": 0.038570855892862445, + "member_std": 0.038954634219408035 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31 + ], + "head_selection_time": "0:00:01.289307" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-best-head-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-best-head-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-best-head-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-best-head-evaluation/1/report.json new file mode 100644 index 000000000..bee7280b0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-best-head-evaluation/1/report.json @@ -0,0 +1,657 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + }, + "mask_stats": { + "keep_rate_actual": 0.9036283493041992, + "min_features_per_member": 256.0, + "mean_features_per_member": 269.28125, + "max_features_per_member": 278.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.916107177734375, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8249481856886859, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9360805899283599, + "recall": 0.9794197976600667, + "f1-score": 0.9572599075643116, + "support": 181971.0 + }, + "1": { + "precision": 0.8896061785166843, + "recall": 0.7126260360339087, + "f1-score": 0.7913416280990652, + "support": 42349.0 + }, + "accuracy": 0.9290522467902995, + "macro avg": { + "precision": 0.9128433842225221, + "recall": 0.8460229168469877, + "f1-score": 0.8743007678316883, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.927306763034311, + "recall": 0.9290522467902995, + "f1-score": 0.9259364713255734, + "support": 224320.0 + }, + "cross-entropy": 0.16230668511159851, + "roc-auc": 0.968832958301438, + "score": 0.968832958301438 + }, + "val": { + "0": { + "precision": 0.9323972370060016, + "recall": 0.9836927304223164, + "f1-score": 0.9573583699096009, + "support": 16741.0 + }, + "1": { + "precision": 0.8897415185783522, + "recall": 0.6485133941713277, + "f1-score": 0.7502128384130768, + "support": 3397.0 + }, + "accuracy": 0.9271526467375112, + "macro avg": { + "precision": 0.9110693777921769, + "recall": 0.8161030622968221, + "f1-score": 0.8537856041613389, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9252018116659121, + "recall": 0.9271526467375112, + "f1-score": 0.9224158050822252, + "support": 20138.0 + }, + "cross-entropy": 0.17814126266590052, + "roc-auc": 0.9574705028701225, + "score": 0.9574705028701225 + }, + "test": { + "0": { + "precision": 0.9257767548906789, + "recall": 0.9790826804594204, + "f1-score": 0.951683856419356, + "support": 13147.0 + }, + "1": { + "precision": 0.8849853617733167, + "recall": 0.6721728081321474, + "f1-score": 0.764036829752663, + "support": 3148.0 + }, + "accuracy": 0.919791347038969, + "macro avg": { + "precision": 0.9053810583319978, + "recall": 0.8256277442957839, + "f1-score": 0.8578603430860094, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9178963433820286, + "recall": 0.919791347038969, + "f1-score": 0.9154326848976162, + "support": 16295.0 + }, + "cross-entropy": 0.18368392811429063, + "roc-auc": 0.9608438143835193, + "score": 0.9608438143835193 + } + }, + "time": "0:03:45.490338", + "diversity": { + "train": { + "mean_centered_corr": 0.09278353303670883, + "mean_pairwise_disagreement": 0.036516382126915464, + "member_std": 0.03687853366136551 + }, + "val": { + "mean_centered_corr": 0.09336636960506439, + "mean_pairwise_disagreement": 0.031406080303967146, + "member_std": 0.03779495880007744 + }, + "test": { + "mean_centered_corr": 0.11713645607233047, + "mean_pairwise_disagreement": 0.03774089617832504, + "member_std": 0.0392426960170269 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 1 + ], + "head_selection_time": "0:00:01.237046" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-best-head-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-best-head-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-best-head-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-best-head-evaluation/2/report.json new file mode 100644 index 000000000..ab5de1ddd --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-best-head-evaluation/2/report.json @@ -0,0 +1,657 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + }, + "mask_stats": { + "keep_rate_actual": 0.8982802033424377, + "min_features_per_member": 254.0, + "mean_features_per_member": 267.6875, + "max_features_per_member": 283.0, + "min_members_per_feature": 22.0, + "mean_members_per_feature": 28.744966506958008, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8155679605741598, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2200, + "metrics": { + "train": { + "0": { + "precision": 0.9481424148606811, + "recall": 0.9693852317127455, + "f1-score": 0.9586461568560319, + "support": 181971.0 + }, + "1": { + "precision": 0.8544366638795987, + "recall": 0.772178799971664, + "f1-score": 0.8112278438620211, + "support": 42349.0 + }, + "accuracy": 0.9321549572039943, + "macro avg": { + "precision": 0.9012895393701399, + "recall": 0.8707820158422048, + "f1-score": 0.8849370003590264, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9304518618636328, + "recall": 0.9321549572039943, + "f1-score": 0.9308152985420904, + "support": 224320.0 + }, + "cross-entropy": 0.15461660323716772, + "roc-auc": 0.9724192435222043, + "score": 0.9724192435222043 + }, + "val": { + "0": { + "precision": 0.9400900380930394, + "recall": 0.9729406845469207, + "f1-score": 0.9562333049578772, + "support": 16741.0 + }, + "1": { + "precision": 0.8389046941678521, + "recall": 0.6944362672946718, + "f1-score": 0.7598647125140925, + "support": 3397.0 + }, + "accuracy": 0.9259608699970205, + "macro avg": { + "precision": 0.8894973661304457, + "recall": 0.8336884759207963, + "f1-score": 0.8580490087359849, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9230214804749114, + "recall": 0.9259608699970205, + "f1-score": 0.9231086595843775, + "support": 20138.0 + }, + "cross-entropy": 0.17667054083105546, + "roc-auc": 0.9571173941905297, + "score": 0.9571173941905297 + }, + "test": { + "0": { + "precision": 0.9367480994907373, + "recall": 0.9653913440328592, + "f1-score": 0.9508540605334133, + "support": 13147.0 + }, + "1": { + "precision": 0.8343044428259286, + "recall": 0.7277636594663278, + "f1-score": 0.7774007465218866, + "support": 3148.0 + }, + "accuracy": 0.9194845044492176, + "macro avg": { + "precision": 0.8855262711583329, + "recall": 0.8465775017495936, + "f1-score": 0.8641274035276499, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9169572046652806, + "recall": 0.9194845044492176, + "f1-score": 0.9173449453135124, + "support": 16295.0 + }, + "cross-entropy": 0.18060997930793665, + "roc-auc": 0.9613875873721536, + "score": 0.9613875873721536 + } + }, + "time": "0:03:57.910375", + "diversity": { + "train": { + "mean_centered_corr": 0.11927956342697144, + "mean_pairwise_disagreement": 0.04567602697471354, + "member_std": 0.04593726247549057 + }, + "val": { + "mean_centered_corr": 0.10152681171894073, + "mean_pairwise_disagreement": 0.042470762224521764, + "member_std": 0.04728136211633682 + }, + "test": { + "mean_centered_corr": 0.12282225489616394, + "mean_pairwise_disagreement": 0.05012644884142177, + "member_std": 0.0499299056828022 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 26 + ], + "head_selection_time": "0:00:01.531446" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-evaluation/0.toml new file mode 100644 index 000000000..10f998b1d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-evaluation/0.toml @@ -0,0 +1,58 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 15 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "homesite-insurance" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-evaluation/0/report.json new file mode 100644 index 000000000..4fb8df0ec --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-evaluation/0/report.json @@ -0,0 +1,653 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.901321291923523, + "min_features_per_member": 256.0, + "mean_features_per_member": 268.59375, + "max_features_per_member": 279.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.842281341552734, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8197222809700668, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9398710962062461, + "recall": 0.9856790367695951, + "f1-score": 0.9622301917057169, + "support": 181971.0 + }, + "1": { + "precision": 0.9221624850657109, + "recall": 0.7290372854140593, + "f1-score": 0.8143058724234792, + "support": 42349.0 + }, + "accuracy": 0.9372280670470756, + "macro avg": { + "precision": 0.9310167906359785, + "recall": 0.8573581610918273, + "f1-score": 0.888268032064598, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.93652791693917, + "recall": 0.9372280670470756, + "f1-score": 0.9343038053055588, + "support": 224320.0 + }, + "cross-entropy": 0.14711870269948546, + "roc-auc": 0.9762437618004894, + "score": 0.9762437618004894 + }, + "val": { + "0": { + "precision": 0.9303347987825499, + "recall": 0.9859626067737889, + "f1-score": 0.9573412986109098, + "support": 16741.0 + }, + "1": { + "precision": 0.9019198664440734, + "recall": 0.6361495437150427, + "f1-score": 0.7460728465389262, + "support": 3397.0 + }, + "accuracy": 0.9269540172807628, + "macro avg": { + "precision": 0.9161273326133117, + "recall": 0.8110560752444158, + "f1-score": 0.8517070725749181, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9255415956266354, + "recall": 0.9269540172807628, + "f1-score": 0.9217032545306373, + "support": 20138.0 + }, + "cross-entropy": 0.175030748761162, + "roc-auc": 0.9589238560635404, + "score": 0.9589238560635404 + }, + "test": { + "0": { + "precision": 0.925214899713467, + "recall": 0.9824294515859131, + "f1-score": 0.9529641789943557, + "support": 13147.0 + }, + "1": { + "precision": 0.9010706638115632, + "recall": 0.6683608640406608, + "f1-score": 0.7674630676636878, + "support": 3148.0 + }, + "accuracy": 0.9217551396133783, + "macro avg": { + "precision": 0.9131427817625151, + "recall": 0.825395157813287, + "f1-score": 0.8602136233290217, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9205505207862382, + "recall": 0.9217551396133783, + "f1-score": 0.9171275727673571, + "support": 16295.0 + }, + "cross-entropy": 0.17966720795996355, + "roc-auc": 0.9624954901031625, + "score": 0.9624954901031625 + } + }, + "time": "0:03:52.600341", + "diversity": { + "train": { + "mean_centered_corr": 0.0775546282529831, + "mean_pairwise_disagreement": 0.0356185432101606, + "member_std": 0.03550577908754349 + }, + "val": { + "mean_centered_corr": 0.08212897181510925, + "mean_pairwise_disagreement": 0.03171593825186856, + "member_std": 0.03616226091980934 + }, + "test": { + "mean_centered_corr": 0.10865671932697296, + "mean_pairwise_disagreement": 0.038570855892862445, + "member_std": 0.038954634219408035 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-evaluation/1.toml new file mode 100644 index 000000000..6083e1ea7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-evaluation/1.toml @@ -0,0 +1,58 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 1 +head_selection = true + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 1 + +[bins] +n_bins = 15 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "homesite-insurance" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-evaluation/1/report.json new file mode 100644 index 000000000..12d2077f8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-evaluation/1/report.json @@ -0,0 +1,653 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + }, + "mask_stats": { + "keep_rate_actual": 0.9036283493041992, + "min_features_per_member": 256.0, + "mean_features_per_member": 269.28125, + "max_features_per_member": 278.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.916107177734375, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8249481856886859, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.937763762727287, + "recall": 0.9854427353809123, + "f1-score": 0.961012234923391, + "support": 181971.0 + }, + "1": { + "precision": 0.9199625343686739, + "recall": 0.7189780160098231, + "f1-score": 0.8071468334968057, + "support": 42349.0 + }, + "accuracy": 0.9351373038516405, + "macro avg": { + "precision": 0.9288631485479805, + "recall": 0.8522103756953677, + "f1-score": 0.8840795342100984, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9344030984095316, + "recall": 0.9351373038516405, + "f1-score": 0.9319642414987546, + "support": 224320.0 + }, + "cross-entropy": 0.1516097613554574, + "roc-auc": 0.9745948569838376, + "score": 0.9745948569838376 + }, + "val": { + "0": { + "precision": 0.9278495227400337, + "recall": 0.9870975449495251, + "f1-score": 0.9565569737489509, + "support": 16741.0 + }, + "1": { + "precision": 0.9072164948453608, + "recall": 0.6217250515160435, + "f1-score": 0.7378165938864629, + "support": 3397.0 + }, + "accuracy": 0.9254642963551495, + "macro avg": { + "precision": 0.9175330087926973, + "recall": 0.8044112982327842, + "f1-score": 0.847186783817707, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.92436901843185, + "recall": 0.9254642963551495, + "f1-score": 0.9196585195631891, + "support": 20138.0 + }, + "cross-entropy": 0.17620349607555666, + "roc-auc": 0.9587836483021374, + "score": 0.9587836483021374 + }, + "test": { + "0": { + "precision": 0.924933744001146, + "recall": 0.9822012626454705, + "f1-score": 0.9527076877674487, + "support": 13147.0 + }, + "1": { + "precision": 0.8997429305912596, + "recall": 0.6670902160101652, + "f1-score": 0.7661437431594309, + "support": 3148.0 + }, + "accuracy": 0.9213255599877263, + "macro avg": { + "precision": 0.9123383372962028, + "recall": 0.8246457393278178, + "f1-score": 0.8594257154634398, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9200671787593956, + "recall": 0.9213255599877263, + "f1-score": 0.9166657548048196, + "support": 16295.0 + }, + "cross-entropy": 0.18052122203960097, + "roc-auc": 0.9621208050227469, + "score": 0.9621208050227469 + } + }, + "time": "0:03:45.490338", + "diversity": { + "train": { + "mean_centered_corr": 0.09278353303670883, + "mean_pairwise_disagreement": 0.036516382126915464, + "member_std": 0.03687853366136551 + }, + "val": { + "mean_centered_corr": 0.09336636960506439, + "mean_pairwise_disagreement": 0.031406080303967146, + "member_std": 0.03779495880007744 + }, + "test": { + "mean_centered_corr": 0.11713645607233047, + "mean_pairwise_disagreement": 0.03774089617832504, + "member_std": 0.0392426960170269 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-evaluation/2.toml new file mode 100644 index 000000000..8f8578ce5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-evaluation/2.toml @@ -0,0 +1,58 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 2 +head_selection = true + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 2 + +[bins] +n_bins = 15 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "homesite-insurance" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-evaluation/2/report.json new file mode 100644 index 000000000..708114571 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-evaluation/2/report.json @@ -0,0 +1,653 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + }, + "mask_stats": { + "keep_rate_actual": 0.8982802033424377, + "min_features_per_member": 254.0, + "mean_features_per_member": 267.6875, + "max_features_per_member": 283.0, + "min_members_per_feature": 22.0, + "mean_members_per_feature": 28.744966506958008, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8155679605741598, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2200, + "metrics": { + "train": { + "0": { + "precision": 0.9447407813455335, + "recall": 0.9849921141280753, + "f1-score": 0.9644466565329101, + "support": 181971.0 + }, + "1": { + "precision": 0.921060238177824, + "recall": 0.752438074098562, + "f1-score": 0.8282539476249269, + "support": 42349.0 + }, + "accuracy": 0.9410886233951498, + "macro avg": { + "precision": 0.9329005097616787, + "recall": 0.8687150941133186, + "f1-score": 0.8963503020789185, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.94027017095587, + "recall": 0.9410886233951498, + "f1-score": 0.9387350613584086, + "support": 224320.0 + }, + "cross-entropy": 0.14547932311545947, + "roc-auc": 0.978555896080379, + "score": 0.978555896080379 + }, + "val": { + "0": { + "precision": 0.9321899586800249, + "recall": 0.9837524640105131, + "f1-score": 0.9572773773541037, + "support": 16741.0 + }, + "1": { + "precision": 0.8899231080534197, + "recall": 0.6473358846040624, + "f1-score": 0.7494887525562373, + "support": 3397.0 + }, + "accuracy": 0.9270036746449498, + "macro avg": { + "precision": 0.9110565333667222, + "recall": 0.8155441743072878, + "f1-score": 0.8533830649551705, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9250601299195432, + "recall": 0.9270036746449498, + "f1-score": 0.9222263316475612, + "support": 20138.0 + }, + "cross-entropy": 0.17711932627215854, + "roc-auc": 0.9585690329227027, + "score": 0.9585690329227027 + }, + "test": { + "0": { + "precision": 0.9298067877559881, + "recall": 0.9773332319160265, + "f1-score": 0.9529778239264258, + "support": 13147.0 + }, + "1": { + "precision": 0.8796445880452343, + "recall": 0.6918678526048284, + "f1-score": 0.7745376955903271, + "support": 3148.0 + }, + "accuracy": 0.9221847192390303, + "macro avg": { + "precision": 0.9047256879006111, + "recall": 0.8346005422604275, + "f1-score": 0.8637577597583765, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9201160479775621, + "recall": 0.9221847192390303, + "f1-score": 0.9185053155494981, + "support": 16295.0 + }, + "cross-entropy": 0.181806240059393, + "roc-auc": 0.9620638665180715, + "score": 0.9620638665180715 + } + }, + "time": "0:03:57.910375", + "diversity": { + "train": { + "mean_centered_corr": 0.11927956342697144, + "mean_pairwise_disagreement": 0.04567602697471354, + "member_std": 0.04593726247549057 + }, + "val": { + "mean_centered_corr": 0.10152681171894073, + "mean_pairwise_disagreement": 0.042470762224521764, + "member_std": 0.04728136211633682 + }, + "test": { + "mean_centered_corr": 0.12282225489616394, + "mean_pairwise_disagreement": 0.05012644884142177, + "member_std": 0.0499299056828022 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..fb5104edb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/0/report.json @@ -0,0 +1,666 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.901321291923523, + "min_features_per_member": 256.0, + "mean_features_per_member": 268.59375, + "max_features_per_member": 279.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.842281341552734, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8197222809700668, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9379184990964076, + "recall": 0.986816580663952, + "f1-score": 0.9617464096253906, + "support": 181971.0 + }, + "1": { + "precision": 0.9269977481589678, + "recall": 0.7193322156367329, + "f1-score": 0.8100676762707583, + "support": 42349.0 + }, + "accuracy": 0.9363186519258203, + "macro avg": { + "precision": 0.9324581236276877, + "recall": 0.8530743981503424, + "f1-score": 0.8859070429480744, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9358567886762504, + "recall": 0.9363186519258203, + "f1-score": 0.9331112336320092, + "support": 224320.0 + }, + "cross-entropy": 0.14950830177513505, + "roc-auc": 0.9756752367295682, + "score": 0.9756752367295682 + }, + "val": { + "0": { + "precision": 0.9291661978170361, + "recall": 0.9865002090675586, + "f1-score": 0.9569752281616688, + "support": 16741.0 + }, + "1": { + "precision": 0.9043993231810491, + "recall": 0.6293788637032676, + "f1-score": 0.7422322513452526, + "support": 3397.0 + }, + "accuracy": 0.9262588141821432, + "macro avg": { + "precision": 0.9167827604990426, + "recall": 0.8079395363854132, + "f1-score": 0.8496037397534607, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9249883711640194, + "recall": 0.9262588141821432, + "f1-score": 0.9207510801705393, + "support": 20138.0 + }, + "cross-entropy": 0.1746148873886249, + "roc-auc": 0.9597570226838343, + "score": 0.9597570226838343 + }, + "test": { + "0": { + "precision": 0.9218071860547847, + "recall": 0.9854719707918156, + "f1-score": 0.9525770163958532, + "support": 13147.0 + }, + "1": { + "precision": 0.9147321428571429, + "recall": 0.6508894536213469, + "f1-score": 0.7605790645879733, + "support": 3148.0 + }, + "accuracy": 0.920834611844124, + "macro avg": { + "precision": 0.9182696644559638, + "recall": 0.8181807122065812, + "f1-score": 0.8565780404919132, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9204403719408739, + "recall": 0.920834611844124, + "f1-score": 0.9154852979367427, + "support": 16295.0 + }, + "cross-entropy": 0.17973653945792817, + "roc-auc": 0.9633062567165207, + "score": 0.9633062567165207 + } + }, + "time": "0:03:52.600341", + "diversity": { + "train": { + "mean_centered_corr": 0.0775546282529831, + "mean_pairwise_disagreement": 0.0356185432101606, + "member_std": 0.03550577908754349 + }, + "val": { + "mean_centered_corr": 0.08212897181510925, + "mean_pairwise_disagreement": 0.03171593825186856, + "member_std": 0.03616226091980934 + }, + "test": { + "mean_centered_corr": 0.10865671932697296, + "mean_pairwise_disagreement": 0.038570855892862445, + "member_std": 0.038954634219408035 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31, + 28, + 30, + 17, + 22, + 12, + 29, + 28, + 18, + 6 + ], + "head_selection_time": "0:00:13.698772" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/1/report.json new file mode 100644 index 000000000..cae1077d0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/1/report.json @@ -0,0 +1,671 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + }, + "mask_stats": { + "keep_rate_actual": 0.9036283493041992, + "min_features_per_member": 256.0, + "mean_features_per_member": 269.28125, + "max_features_per_member": 278.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.916107177734375, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8249481856886859, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9365119066228962, + "recall": 0.9876628693583044, + "f1-score": 0.9614075066665597, + "support": 181971.0 + }, + "1": { + "precision": 0.9307312557852515, + "recall": 0.7122954497154597, + "f1-score": 0.8069931379499459, + "support": 42349.0 + }, + "accuracy": 0.9356767118402283, + "macro avg": { + "precision": 0.9336215812040738, + "recall": 0.849979159536882, + "f1-score": 0.8842003223082529, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9354205871581875, + "recall": 0.9356767118402283, + "f1-score": 0.9322558746195738, + "support": 224320.0 + }, + "cross-entropy": 0.15071028513275977, + "roc-auc": 0.9752016380773885, + "score": 0.9752016380773885 + }, + "val": { + "0": { + "precision": 0.9268197088465846, + "recall": 0.9887700854190311, + "f1-score": 0.9567931562671599, + "support": 16741.0 + }, + "1": { + "precision": 0.9174714661984197, + "recall": 0.6152487488960848, + "f1-score": 0.7365638766519824, + "support": 3397.0 + }, + "accuracy": 0.9257622405402721, + "macro avg": { + "precision": 0.9221455875225022, + "recall": 0.802009417157558, + "f1-score": 0.8466785164595712, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9252427905689098, + "recall": 0.9257622405402721, + "f1-score": 0.9196435454392347, + "support": 20138.0 + }, + "cross-entropy": 0.1778182330217696, + "roc-auc": 0.9595148440428459, + "score": 0.9595148440428459 + }, + "test": { + "0": { + "precision": 0.9237536656891495, + "recall": 0.9823533886057656, + "f1-score": 0.9521527572987319, + "support": 13147.0 + }, + "1": { + "precision": 0.8997407087294728, + "recall": 0.6613722998729352, + "f1-score": 0.7623581105822044, + "support": 3148.0 + }, + "accuracy": 0.9203436637005217, + "macro avg": { + "precision": 0.9117471872093111, + "recall": 0.8218628442393504, + "f1-score": 0.8572554339404681, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9191146482906185, + "recall": 0.9203436637005217, + "f1-score": 0.9154866911518384, + "support": 16295.0 + }, + "cross-entropy": 0.1817569672472227, + "roc-auc": 0.9624332721317901, + "score": 0.9624332721317901 + } + }, + "time": "0:03:45.490338", + "diversity": { + "train": { + "mean_centered_corr": 0.09278353303670883, + "mean_pairwise_disagreement": 0.036516382126915464, + "member_std": 0.03687853366136551 + }, + "val": { + "mean_centered_corr": 0.09336636960506439, + "mean_pairwise_disagreement": 0.031406080303967146, + "member_std": 0.03779495880007744 + }, + "test": { + "mean_centered_corr": 0.11713645607233047, + "mean_pairwise_disagreement": 0.03774089617832504, + "member_std": 0.0392426960170269 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 1, + 17, + 6, + 16, + 11, + 27, + 13, + 25, + 26, + 21, + 4, + 15, + 17, + 7, + 6 + ], + "head_selection_time": "0:00:16.127976" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/2/report.json new file mode 100644 index 000000000..3ca863cda --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/2/report.json @@ -0,0 +1,667 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + }, + "mask_stats": { + "keep_rate_actual": 0.8982802033424377, + "min_features_per_member": 254.0, + "mean_features_per_member": 267.6875, + "max_features_per_member": 283.0, + "min_members_per_feature": 22.0, + "mean_members_per_feature": 28.744966506958008, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8155679605741598, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2200, + "metrics": { + "train": { + "0": { + "precision": 0.9425560252475534, + "recall": 0.9839315055695688, + "f1-score": 0.9627994515096927, + "support": 181971.0 + }, + "1": { + "precision": 0.9149035243444603, + "recall": 0.7423315780774045, + "f1-score": 0.8196323816973016, + "support": 42349.0 + }, + "accuracy": 0.9383202567760343, + "macro avg": { + "precision": 0.9287297747960068, + "recall": 0.8631315418234866, + "f1-score": 0.8912159166034972, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9373355555580692, + "recall": 0.9383202567760343, + "f1-score": 0.935771178330819, + "support": 224320.0 + }, + "cross-entropy": 0.14788625943279124, + "roc-auc": 0.9766724082765327, + "score": 0.9766724082765327 + }, + "val": { + "0": { + "precision": 0.9320020365446625, + "recall": 0.984110865539693, + "f1-score": 0.9573478993549887, + "support": 16741.0 + }, + "1": { + "precision": 0.8919138561560341, + "recall": 0.6461583750367972, + "f1-score": 0.7494025264595425, + "support": 3397.0 + }, + "accuracy": 0.927102989373324, + "macro avg": { + "precision": 0.9119579463503483, + "recall": 0.815134620288245, + "f1-score": 0.8533752129072656, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9252397190960494, + "recall": 0.927102989373324, + "f1-score": 0.92227041242849, + "support": 20138.0 + }, + "cross-entropy": 0.17472041508549696, + "roc-auc": 0.9596270313530297, + "score": 0.9596270313530297 + }, + "test": { + "0": { + "precision": 0.9284322920424212, + "recall": 0.9788544915189777, + "f1-score": 0.9529768957345972, + "support": 13147.0 + }, + "1": { + "precision": 0.885784716516023, + "recall": 0.6848792884371029, + "f1-score": 0.7724829810103906, + "support": 3148.0 + }, + "accuracy": 0.9220619822031298, + "macro avg": { + "precision": 0.9071085042792221, + "recall": 0.8318668899780404, + "f1-score": 0.8627299383724939, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9201932881911109, + "recall": 0.9220619822031298, + "f1-score": 0.9181076202788253, + "support": 16295.0 + }, + "cross-entropy": 0.17945947055912823, + "roc-auc": 0.9628075343716237, + "score": 0.9628075343716237 + } + }, + "time": "0:03:57.910375", + "diversity": { + "train": { + "mean_centered_corr": 0.11927956342697144, + "mean_pairwise_disagreement": 0.04567602697471354, + "member_std": 0.04593726247549057 + }, + "val": { + "mean_centered_corr": 0.10152681171894073, + "mean_pairwise_disagreement": 0.042470762224521764, + "member_std": 0.04728136211633682 + }, + "test": { + "mean_centered_corr": 0.12282225489616394, + "mean_pairwise_disagreement": 0.05012644884142177, + "member_std": 0.0499299056828022 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 26, + 5, + 23, + 31, + 4, + 30, + 16, + 14, + 0, + 25, + 8 + ], + "head_selection_time": "0:00:22.293435" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..f143c209a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-best-head-evaluation/0/report.json @@ -0,0 +1,272 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/esam_only-evaluation/_esam_diagnostics/esam_only-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9545321160743397, + "recall": 0.9652801820070231, + "f1-score": 0.9598760625261277, + "support": 181971.0 + }, + "1": { + "precision": 0.8432258064516129, + "recall": 0.8024274481097546, + "f1-score": 0.8223208992244311, + "support": 42349.0 + }, + "accuracy": 0.934535485021398, + "macro avg": { + "precision": 0.8988789612629764, + "recall": 0.8838538150583889, + "f1-score": 0.8910984808752793, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9335187828619073, + "recall": 0.934535485021398, + "f1-score": 0.9339072518509157, + "support": 224320.0 + }, + "cross-entropy": 0.1498442336366005, + "roc-auc": 0.9747257011404723, + "score": 0.9747257011404723 + }, + "val": { + "0": { + "precision": 0.9430700273589848, + "recall": 0.9677438623738128, + "f1-score": 0.955247641509434, + "support": 16741.0 + }, + "1": { + "precision": 0.8175059141601893, + "recall": 0.7120989108036503, + "f1-score": 0.7611705475141598, + "support": 3397.0 + }, + "accuracy": 0.9246201211639686, + "macro avg": { + "precision": 0.880287970759587, + "recall": 0.8399213865887316, + "f1-score": 0.8582090945117968, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9218891110546691, + "recall": 0.9246201211639686, + "f1-score": 0.9225095399947877, + "support": 20138.0 + }, + "cross-entropy": 0.18072654373317865, + "roc-auc": 0.9561512908829329, + "score": 0.9561512908829329 + }, + "test": { + "0": { + "precision": 0.938474098263322, + "recall": 0.9618163839659238, + "f1-score": 0.9500018782164457, + "support": 13147.0 + }, + "1": { + "precision": 0.8220489188231124, + "recall": 0.7366581956797967, + "f1-score": 0.7770145753057464, + "support": 3148.0 + }, + "accuracy": 0.9183185026081621, + "macro avg": { + "precision": 0.8802615085432172, + "recall": 0.8492372898228603, + "f1-score": 0.8635082267610961, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9159821396945721, + "recall": 0.9183185026081621, + "f1-score": 0.9165827907931329, + "support": 16295.0 + }, + "cross-entropy": 0.18568286621420937, + "roc-auc": 0.9606910602029306, + "score": 0.9606910602029306 + } + }, + "time": "0:05:02.257608", + "diversity": { + "train": { + "mean_centered_corr": 0.049995824694633484, + "mean_pairwise_disagreement": 0.03397909919872533, + "member_std": 0.031108465045690536 + }, + "val": { + "mean_centered_corr": 0.05969587340950966, + "mean_pairwise_disagreement": 0.03266793800197989, + "member_std": 0.03460051864385605 + }, + "test": { + "mean_centered_corr": 0.07376471161842346, + "mean_pairwise_disagreement": 0.04182474833958566, + "member_std": 0.03703681379556656 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 17 + ], + "head_selection_time": "0:00:00.953794" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-best-head-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-best-head-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-best-head-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-best-head-evaluation/1/report.json new file mode 100644 index 000000000..256963311 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-best-head-evaluation/1/report.json @@ -0,0 +1,272 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__1.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.941301001780827, + "recall": 0.9847008589280709, + "f1-score": 0.9625119516990213, + "support": 181971.0 + }, + "1": { + "precision": 0.9180187873612298, + "recall": 0.7361448912607146, + "f1-score": 0.8170833988572627, + "support": 42349.0 + }, + "accuracy": 0.9377763908701855, + "macro avg": { + "precision": 0.9296598945710284, + "recall": 0.8604228750943927, + "f1-score": 0.889797675278142, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9369055912135325, + "recall": 0.9377763908701855, + "f1-score": 0.9350567368974181, + "support": 224320.0 + }, + "cross-entropy": 0.14229130006796945, + "roc-auc": 0.9765326990887259, + "score": 0.9765326990887259 + }, + "val": { + "0": { + "precision": 0.9301342054809969, + "recall": 0.9853055373036258, + "f1-score": 0.9569253081943437, + "support": 16741.0 + }, + "1": { + "precision": 0.8976705490848585, + "recall": 0.6352664115395937, + "f1-score": 0.7440096535080158, + "support": 3397.0 + }, + "accuracy": 0.9262588141821432, + "macro avg": { + "precision": 0.9139023772829278, + "recall": 0.8102859744216098, + "f1-score": 0.8504674808511798, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9246580389908946, + "recall": 0.9262588141821432, + "f1-score": 0.921009403984916, + "support": 20138.0 + }, + "cross-entropy": 0.18049319389718857, + "roc-auc": 0.9562486459053205, + "score": 0.9562486459053205 + }, + "test": { + "0": { + "precision": 0.9297238372093023, + "recall": 0.973073705027763, + "f1-score": 0.9509049689671832, + "support": 13147.0 + }, + "1": { + "precision": 0.8603550295857988, + "recall": 0.6928208386277002, + "f1-score": 0.7675523491113848, + "support": 3148.0 + }, + "accuracy": 0.918932187787665, + "macro avg": { + "precision": 0.8950394333975505, + "recall": 0.8329472718277315, + "f1-score": 0.859228659039284, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.916322609446259, + "recall": 0.918932187787665, + "f1-score": 0.9154834257142802, + "support": 16295.0 + }, + "cross-entropy": 0.18623170204474931, + "roc-auc": 0.9594156280332771, + "score": 0.9594156280332771 + } + }, + "time": "0:04:54.388848", + "diversity": { + "train": { + "mean_centered_corr": 0.046915579587221146, + "mean_pairwise_disagreement": 0.03045741646438268, + "member_std": 0.02836863324046135 + }, + "val": { + "mean_centered_corr": 0.05156576633453369, + "mean_pairwise_disagreement": 0.027064564985471218, + "member_std": 0.029862722381949425 + }, + "test": { + "mean_centered_corr": 0.06818301975727081, + "mean_pairwise_disagreement": 0.03394842075047758, + "member_std": 0.03134872391819954 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 27 + ], + "head_selection_time": "0:00:01.014267" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-best-head-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-best-head-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-best-head-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-best-head-evaluation/2/report.json new file mode 100644 index 000000000..1019d433f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-best-head-evaluation/2/report.json @@ -0,0 +1,272 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__2.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9469624787748779, + "recall": 0.9776447895543795, + "f1-score": 0.9620590636981597, + "support": 181971.0 + }, + "1": { + "precision": 0.8884042465640688, + "recall": 0.7647169944980992, + "f1-score": 0.8219334534656481, + "support": 42349.0 + }, + "accuracy": 0.9374465049928673, + "macro avg": { + "precision": 0.9176833626694734, + "recall": 0.8711808920262394, + "f1-score": 0.8919962585819039, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9359073674344021, + "recall": 0.9374465049928673, + "f1-score": 0.9356049826187347, + "support": 224320.0 + }, + "cross-entropy": 0.14571912513044444, + "roc-auc": 0.975172867812646, + "score": 0.975172867812646 + }, + "val": { + "0": { + "precision": 0.9367074080449876, + "recall": 0.9750910937219999, + "f1-score": 0.9555139311636619, + "support": 16741.0 + }, + "1": { + "precision": 0.8461822205828108, + "recall": 0.6753017368266118, + "f1-score": 0.7511460379829732, + "support": 3397.0 + }, + "accuracy": 0.9245208064355944, + "macro avg": { + "precision": 0.8914448143138992, + "recall": 0.8251964152743059, + "f1-score": 0.8533299845733175, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9214370702850803, + "recall": 0.9245208064355944, + "f1-score": 0.9210399152169543, + "support": 20138.0 + }, + "cross-entropy": 0.18217191919121103, + "roc-auc": 0.9550014764588559, + "score": 0.9550014764588559 + }, + "test": { + "0": { + "precision": 0.9363965428085986, + "recall": 0.9641743363504982, + "f1-score": 0.9500824464098336, + "support": 13147.0 + }, + "1": { + "precision": 0.8292240754169689, + "recall": 0.7264930114358322, + "f1-score": 0.7744666440907552, + "support": 3148.0 + }, + "accuracy": 0.9182571340902117, + "macro avg": { + "precision": 0.8828103091127837, + "recall": 0.8453336738931652, + "f1-score": 0.8622745452502945, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9156920980495405, + "recall": 0.9182571340902117, + "f1-score": 0.9161555641944019, + "support": 16295.0 + }, + "cross-entropy": 0.188877181233535, + "roc-auc": 0.9593146174587831, + "score": 0.9593146174587831 + } + }, + "time": "0:05:03.267508", + "diversity": { + "train": { + "mean_centered_corr": 0.057024113833904266, + "mean_pairwise_disagreement": 0.032304333383185314, + "member_std": 0.030768897384405136 + }, + "val": { + "mean_centered_corr": 0.06899179518222809, + "mean_pairwise_disagreement": 0.0302923937732869, + "member_std": 0.03341741859912872 + }, + "test": { + "mean_centered_corr": 0.08463822305202484, + "mean_pairwise_disagreement": 0.03738159340387413, + "member_std": 0.035034921020269394 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 16 + ], + "head_selection_time": "0:00:01.057452" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-evaluation/0.toml new file mode 100644 index 000000000..016b99992 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-evaluation/0.toml @@ -0,0 +1,44 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-evaluation/0/report.json new file mode 100644 index 000000000..b0473609b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-evaluation/0/report.json @@ -0,0 +1,268 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/esam_only-evaluation/_esam_diagnostics/esam_only-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9496102021429594, + "recall": 0.9833215182638992, + "f1-score": 0.9661718889206862, + "support": 181971.0 + }, + "1": { + "precision": 0.9154336983476832, + "recall": 0.7757916361661432, + "f1-score": 0.8398476443671873, + "support": 42349.0 + }, + "accuracy": 0.9441422967189729, + "macro avg": { + "precision": 0.9325219502453213, + "recall": 0.8795565772150212, + "f1-score": 0.9030097666439367, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9431580767897758, + "recall": 0.9441422967189729, + "f1-score": 0.9423233447311528, + "support": 224320.0 + }, + "cross-entropy": 0.13329761494760606, + "roc-auc": 0.9810524262916843, + "score": 0.9810524262916843 + }, + "val": { + "0": { + "precision": 0.936472602739726, + "recall": 0.9800489815423212, + "f1-score": 0.9577653892180613, + "support": 16741.0 + }, + "1": { + "precision": 0.8724216959511077, + "recall": 0.6723579629084486, + "f1-score": 0.7594347464671655, + "support": 3397.0 + }, + "accuracy": 0.9281457940212533, + "macro avg": { + "precision": 0.9044471493454169, + "recall": 0.8262034722253849, + "f1-score": 0.8586000678426133, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9256681072406231, + "recall": 0.9281457940212533, + "f1-score": 0.9243097732966793, + "support": 20138.0 + }, + "cross-entropy": 0.1734408298186842, + "roc-auc": 0.9583120395781356, + "score": 0.9583120395781356 + }, + "test": { + "0": { + "precision": 0.9324796274738067, + "recall": 0.9748231535711569, + "f1-score": 0.9531813617939088, + "support": 13147.0 + }, + "1": { + "precision": 0.8702469619756958, + "recall": 0.7052096569250318, + "f1-score": 0.779084049833304, + "support": 3148.0 + }, + "accuracy": 0.922737035900583, + "macro avg": { + "precision": 0.9013632947247513, + "recall": 0.8400164052480943, + "f1-score": 0.8661327058136064, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9204570174101029, + "recall": 0.922737035900583, + "f1-score": 0.9195478338373586, + "support": 16295.0 + }, + "cross-entropy": 0.17694458755574416, + "roc-auc": 0.9627483366901237, + "score": 0.9627483366901237 + } + }, + "time": "0:05:02.257608", + "diversity": { + "train": { + "mean_centered_corr": 0.049995824694633484, + "mean_pairwise_disagreement": 0.03397909919872533, + "member_std": 0.031108465045690536 + }, + "val": { + "mean_centered_corr": 0.05969587340950966, + "mean_pairwise_disagreement": 0.03266793800197989, + "member_std": 0.03460051864385605 + }, + "test": { + "mean_centered_corr": 0.07376471161842346, + "mean_pairwise_disagreement": 0.04182474833958566, + "member_std": 0.03703681379556656 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-evaluation/1.toml new file mode 100644 index 000000000..d6bf43167 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-evaluation/1.toml @@ -0,0 +1,44 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 1 +head_selection = true +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-evaluation/1/report.json new file mode 100644 index 000000000..5c49bdfdd --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-evaluation/1/report.json @@ -0,0 +1,268 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__1.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9423287951737644, + "recall": 0.9862835286941326, + "f1-score": 0.9638052788443466, + "support": 181971.0 + }, + "1": { + "precision": 0.9262868787100204, + "recall": 0.7406314198682378, + "f1-score": 0.823120325416612, + "support": 42349.0 + }, + "accuracy": 0.9399072753209701, + "macro avg": { + "precision": 0.9343078369418925, + "recall": 0.8634574742811851, + "f1-score": 0.8934628021304793, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9393002684248205, + "recall": 0.9399072753209701, + "f1-score": 0.9372456002926745, + "support": 224320.0 + }, + "cross-entropy": 0.13937913474355193, + "roc-auc": 0.9786839612084102, + "score": 0.9786839612084102 + }, + "val": { + "0": { + "precision": 0.930703684893629, + "recall": 0.9851860701272326, + "f1-score": 0.9571702164703151, + "support": 16741.0 + }, + "1": { + "precision": 0.8973934629706247, + "recall": 0.6385045628495731, + "f1-score": 0.7461300309597523, + "support": 3397.0 + }, + "accuracy": 0.9267057304598272, + "macro avg": { + "precision": 0.9140485739321269, + "recall": 0.8118453164884029, + "f1-score": 0.8516501237150337, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9250847145950668, + "recall": 0.9267057304598272, + "f1-score": 0.9215706777783208, + "support": 20138.0 + }, + "cross-entropy": 0.17362094113783183, + "roc-auc": 0.9584753793781823, + "score": 0.9584753793781823 + }, + "test": { + "0": { + "precision": 0.927138027763792, + "recall": 0.9804518141020765, + "f1-score": 0.9530499075785582, + "support": 13147.0 + }, + "1": { + "precision": 0.8925585284280937, + "recall": 0.6782083862770013, + "f1-score": 0.7707581227436823, + "support": 3148.0 + }, + "accuracy": 0.9220619822031298, + "macro avg": { + "precision": 0.9098482780959428, + "recall": 0.8293301001895389, + "f1-score": 0.8619040151611203, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9204576801781045, + "recall": 0.9220619822031298, + "f1-score": 0.9178333050219342, + "support": 16295.0 + }, + "cross-entropy": 0.17768448825468044, + "roc-auc": 0.9624917932683585, + "score": 0.9624917932683585 + } + }, + "time": "0:04:54.388848", + "diversity": { + "train": { + "mean_centered_corr": 0.046915579587221146, + "mean_pairwise_disagreement": 0.03045741646438268, + "member_std": 0.02836863324046135 + }, + "val": { + "mean_centered_corr": 0.05156576633453369, + "mean_pairwise_disagreement": 0.027064564985471218, + "member_std": 0.029862722381949425 + }, + "test": { + "mean_centered_corr": 0.06818301975727081, + "mean_pairwise_disagreement": 0.03394842075047758, + "member_std": 0.03134872391819954 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-evaluation/2.toml new file mode 100644 index 000000000..856c5b853 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-evaluation/2.toml @@ -0,0 +1,44 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 2 +head_selection = true +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-evaluation/2/report.json new file mode 100644 index 000000000..3ab8d3d63 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-evaluation/2/report.json @@ -0,0 +1,268 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__2.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.947000732878843, + "recall": 0.9870309005281062, + "f1-score": 0.9666015488356824, + "support": 181971.0 + }, + "1": { + "precision": 0.9319040886401015, + "recall": 0.762639023353562, + "f1-score": 0.8388177544606914, + "support": 42349.0 + }, + "accuracy": 0.9446683309557775, + "macro avg": { + "precision": 0.9394524107594723, + "recall": 0.8748349619408341, + "f1-score": 0.902709651648187, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9441506625022985, + "recall": 0.9446683309557775, + "f1-score": 0.9424774586609922, + "support": 224320.0 + }, + "cross-entropy": 0.13143490246529854, + "roc-auc": 0.981694255249284, + "score": 0.981694255249284 + }, + "val": { + "0": { + "precision": 0.9317641733620007, + "recall": 0.9836927304223164, + "f1-score": 0.9570245532471306, + "support": 16741.0 + }, + "1": { + "precision": 0.8892045454545454, + "recall": 0.6449808654695319, + "f1-score": 0.7476539839617813, + "support": 3397.0 + }, + "accuracy": 0.9265567583672659, + "macro avg": { + "precision": 0.910484359408273, + "recall": 0.8143367979459242, + "f1-score": 0.852339268604456, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9245849571537563, + "recall": 0.9265567583672659, + "f1-score": 0.9217066555481371, + "support": 20138.0 + }, + "cross-entropy": 0.17442511117528403, + "roc-auc": 0.9582681142018287, + "score": 0.9582681142018287 + }, + "test": { + "0": { + "precision": 0.9284271284271284, + "recall": 0.9787784285388301, + "f1-score": 0.9529381271522198, + "support": 13147.0 + }, + "1": { + "precision": 0.8854209445585216, + "recall": 0.6848792884371029, + "f1-score": 0.7723446175891098, + "support": 3148.0 + }, + "accuracy": 0.9220006136851795, + "macro avg": { + "precision": 0.906924036492825, + "recall": 0.8318288584879665, + "f1-score": 0.8626413723706647, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9201188457135123, + "recall": 0.9220006136851795, + "f1-score": 0.9180496111592974, + "support": 16295.0 + }, + "cross-entropy": 0.17762575038093437, + "roc-auc": 0.9627356393915001, + "score": 0.9627356393915001 + } + }, + "time": "0:05:03.267508", + "diversity": { + "train": { + "mean_centered_corr": 0.057024113833904266, + "mean_pairwise_disagreement": 0.032304333383185314, + "member_std": 0.030768897384405136 + }, + "val": { + "mean_centered_corr": 0.06899179518222809, + "mean_pairwise_disagreement": 0.0302923937732869, + "member_std": 0.03341741859912872 + }, + "test": { + "mean_centered_corr": 0.08463822305202484, + "mean_pairwise_disagreement": 0.03738159340387413, + "member_std": 0.035034921020269394 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..55f982fc5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,290 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/esam_only-evaluation/_esam_diagnostics/esam_only-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9495084683651196, + "recall": 0.9824752295695468, + "f1-score": 0.9657105817533626, + "support": 181971.0 + }, + "1": { + "precision": 0.9114928811301379, + "recall": 0.7755082764646155, + "f1-score": 0.8380199030364889, + "support": 42349.0 + }, + "accuracy": 0.9434022824536377, + "macro avg": { + "precision": 0.9305006747476288, + "recall": 0.8789917530170812, + "f1-score": 0.9018652423949257, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9423315688295711, + "recall": 0.9434022824536377, + "f1-score": 0.9416040707290184, + "support": 224320.0 + }, + "cross-entropy": 0.13410595787789584, + "roc-auc": 0.9804656245919036, + "score": 0.9804656245919036 + }, + "val": { + "0": { + "precision": 0.9362795477903392, + "recall": 0.9795113792485515, + "f1-score": 0.957407677711283, + "support": 16741.0 + }, + "1": { + "precision": 0.8692835365853658, + "recall": 0.6714748307329997, + "f1-score": 0.7576814482644079, + "support": 3397.0 + }, + "accuracy": 0.927549905651008, + "macro avg": { + "precision": 0.9027815421878524, + "recall": 0.8254931049907757, + "f1-score": 0.8575445629878454, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9249782542128591, + "recall": 0.927549905651008, + "f1-score": 0.9237166457601939, + "support": 20138.0 + }, + "cross-entropy": 0.17278545942960094, + "roc-auc": 0.958991133281215, + "score": 0.958991133281215 + }, + "test": { + "0": { + "precision": 0.9319371727748691, + "recall": 0.9748231535711569, + "f1-score": 0.9528978772445073, + "support": 13147.0 + }, + "1": { + "precision": 0.8698387731026347, + "recall": 0.7026683608640406, + "f1-score": 0.7773677736777368, + "support": 3148.0 + }, + "accuracy": 0.9222460877569807, + "macro avg": { + "precision": 0.9008879729387519, + "recall": 0.8387457572175987, + "f1-score": 0.865132825461122, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9199405012702239, + "recall": 0.9222460877569807, + "f1-score": 0.9189875510077357, + "support": 16295.0 + }, + "cross-entropy": 0.1769733127255982, + "roc-auc": 0.9627938464179217, + "score": 0.9627938464179217 + } + }, + "time": "0:05:02.257608", + "diversity": { + "train": { + "mean_centered_corr": 0.049995824694633484, + "mean_pairwise_disagreement": 0.03397909919872533, + "member_std": 0.031108465045690536 + }, + "val": { + "mean_centered_corr": 0.05969587340950966, + "mean_pairwise_disagreement": 0.03266793800197989, + "member_std": 0.03460051864385605 + }, + "test": { + "mean_centered_corr": 0.07376471161842346, + "mean_pairwise_disagreement": 0.04182474833958566, + "member_std": 0.03703681379556656 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 17, + 23, + 5, + 26, + 21, + 24, + 30, + 9, + 28, + 15, + 22, + 12, + 20, + 29, + 23, + 26, + 5, + 21, + 30 + ], + "head_selection_time": "0:00:17.134347" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/1/report.json new file mode 100644 index 000000000..2050513dd --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/1/report.json @@ -0,0 +1,278 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__1.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9430724174373912, + "recall": 0.9856625506261987, + "f1-score": 0.9638972484952708, + "support": 181971.0 + }, + "1": { + "precision": 0.9235592276815798, + "recall": 0.7443387092965595, + "f1-score": 0.8243200836820084, + "support": 42349.0 + }, + "accuracy": 0.9401034236804565, + "macro avg": { + "precision": 0.9333158225594855, + "recall": 0.865000629961379, + "f1-score": 0.8941086660886396, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9393885547725871, + "recall": 0.9401034236804565, + "f1-score": 0.9375467075150781, + "support": 224320.0 + }, + "cross-entropy": 0.13900322030342313, + "roc-auc": 0.978510242788649, + "score": 0.978510242788649 + }, + "val": { + "0": { + "precision": 0.9310831404865384, + "recall": 0.9853652708918225, + "f1-score": 0.9574554530152649, + "support": 16741.0 + }, + "1": { + "precision": 0.8988021478727798, + "recall": 0.6405652045922873, + "f1-score": 0.7480233757304916, + "support": 3397.0 + }, + "accuracy": 0.9272023041016982, + "macro avg": { + "precision": 0.9149426441796591, + "recall": 0.8129652377420549, + "f1-score": 0.8527394143728783, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9256377868313125, + "recall": 0.9272023041016982, + "f1-score": 0.9221271797738122, + "support": 20138.0 + }, + "cross-entropy": 0.17295465116447717, + "roc-auc": 0.9591595812965608, + "score": 0.9591595812965608 + }, + "test": { + "0": { + "precision": 0.9273710782545979, + "recall": 0.9780177987373545, + "f1-score": 0.952021323856064, + "support": 13147.0 + }, + "1": { + "precision": 0.8810699588477366, + "recall": 0.6801143583227446, + "f1-score": 0.767658659017569, + "support": 3148.0 + }, + "accuracy": 0.9204664007364222, + "macro avg": { + "precision": 0.9042205185511673, + "recall": 0.8290660785300495, + "f1-score": 0.8598399914368164, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9184262532228213, + "recall": 0.9204664007364222, + "f1-score": 0.916404651937587, + "support": 16295.0 + }, + "cross-entropy": 0.1781920882065017, + "roc-auc": 0.9622548696496048, + "score": 0.9622548696496048 + } + }, + "time": "0:04:54.388848", + "diversity": { + "train": { + "mean_centered_corr": 0.046915579587221146, + "mean_pairwise_disagreement": 0.03045741646438268, + "member_std": 0.02836863324046135 + }, + "val": { + "mean_centered_corr": 0.05156576633453369, + "mean_pairwise_disagreement": 0.027064564985471218, + "member_std": 0.029862722381949425 + }, + "test": { + "mean_centered_corr": 0.06818301975727081, + "mean_pairwise_disagreement": 0.03394842075047758, + "member_std": 0.03134872391819954 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 27, + 14, + 18, + 11, + 24, + 5, + 26 + ], + "head_selection_time": "0:00:08.881424" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/2/report.json new file mode 100644 index 000000000..8a5d00460 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/2/report.json @@ -0,0 +1,283 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__2.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.945536038382021, + "recall": 0.987717823169626, + "f1-score": 0.9661667472988228, + "support": 181971.0 + }, + "1": { + "precision": 0.9347083053372673, + "recall": 0.7555314175069069, + "f1-score": 0.8356228780360407, + "support": 42349.0 + }, + "accuracy": 0.9438837375178317, + "macro avg": { + "precision": 0.9401221718596442, + "recall": 0.8716246203382665, + "f1-score": 0.9008948126674318, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9434918886552366, + "recall": 0.9438837375178317, + "f1-score": 0.9415215871730669, + "support": 224320.0 + }, + "cross-entropy": 0.13196222295418844, + "roc-auc": 0.9811511759509819, + "score": 0.9811511759509819 + }, + "val": { + "0": { + "precision": 0.9318708656075083, + "recall": 0.9845290006570695, + "f1-score": 0.9574764726385501, + "support": 16741.0 + }, + "1": { + "precision": 0.8943288453692371, + "recall": 0.6452752428613483, + "f1-score": 0.7496580027359782, + "support": 3397.0 + }, + "accuracy": 0.9273016188300724, + "macro avg": { + "precision": 0.9130998554883727, + "recall": 0.814902121759209, + "f1-score": 0.8535672376872641, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9255380498984306, + "recall": 0.9273016188300724, + "f1-score": 0.9224203924787012, + "support": 20138.0 + }, + "cross-entropy": 0.17394704368189032, + "roc-auc": 0.9590920051471822, + "score": 0.9590920051471822 + }, + "test": { + "0": { + "precision": 0.9284375900835976, + "recall": 0.9799193732410436, + "f1-score": 0.9534840691262998, + "support": 13147.0 + }, + "1": { + "precision": 0.8908639933856966, + "recall": 0.684561626429479, + "f1-score": 0.7742051374169211, + "support": 3148.0 + }, + "accuracy": 0.9228597729364836, + "macro avg": { + "precision": 0.909650791734647, + "recall": 0.8322404998352613, + "f1-score": 0.8638446032716105, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9211788185337362, + "recall": 0.9228597729364836, + "f1-score": 0.9188495139240216, + "support": 16295.0 + }, + "cross-entropy": 0.1774981138265784, + "roc-auc": 0.9631468941416912, + "score": 0.9631468941416912 + } + }, + "time": "0:05:03.267508", + "diversity": { + "train": { + "mean_centered_corr": 0.057024113833904266, + "mean_pairwise_disagreement": 0.032304333383185314, + "member_std": 0.030768897384405136 + }, + "val": { + "mean_centered_corr": 0.06899179518222809, + "mean_pairwise_disagreement": 0.0302923937732869, + "member_std": 0.03341741859912872 + }, + "test": { + "mean_centered_corr": 0.08463822305202484, + "mean_pairwise_disagreement": 0.03738159340387413, + "member_std": 0.035034921020269394 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 16, + 1, + 5, + 10, + 3, + 4, + 27, + 15, + 18, + 1, + 6, + 25 + ], + "head_selection_time": "0:00:12.525387" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..a5901fec9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-best-head-evaluation/0/report.json @@ -0,0 +1,298 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9506081938743591, + "min_features_per_member": 275.0, + "mean_features_per_member": 283.28125, + "max_features_per_member": 292.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.419462203979492, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9056525652610176, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9539299644574761, + "recall": 0.9690170411768908, + "f1-score": 0.9614143176489831, + "support": 181971.0 + }, + "1": { + "precision": 0.8571609536115122, + "recall": 0.7989090651491181, + "f1-score": 0.8270105108775361, + "support": 42349.0 + }, + "accuracy": 0.9369026390870185, + "macro avg": { + "precision": 0.9055454590344942, + "recall": 0.8839630531630045, + "f1-score": 0.8942124142632596, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9356611037214039, + "recall": 0.9369026390870185, + "f1-score": 0.9360404463358412, + "support": 224320.0 + }, + "cross-entropy": 0.14460702857706767, + "roc-auc": 0.9762667563935794, + "score": 0.9762667563935794 + }, + "val": { + "0": { + "precision": 0.9437383395522388, + "recall": 0.9669075921390597, + "f1-score": 0.9551824860590682, + "support": 16741.0 + }, + "1": { + "precision": 0.8144675150703282, + "recall": 0.7159258168972623, + "f1-score": 0.7620241265862447, + "support": 3397.0 + }, + "accuracy": 0.9245704637997815, + "macro avg": { + "precision": 0.8791029273112836, + "recall": 0.8414167045181611, + "f1-score": 0.8586033063226565, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9219321527032445, + "recall": 0.9245704637997815, + "f1-score": 0.9225993622568445, + "support": 20138.0 + }, + "cross-entropy": 0.1811185402192146, + "roc-auc": 0.9563975490624737, + "score": 0.9563975490624737 + }, + "test": { + "0": { + "precision": 0.9422366337380288, + "recall": 0.9504069369437894, + "f1-score": 0.9463041502574977, + "support": 13147.0 + }, + "1": { + "precision": 0.7851021753460777, + "recall": 0.7566709021601017, + "f1-score": 0.7706243934001941, + "support": 3148.0 + }, + "accuracy": 0.9129794415464867, + "macro avg": { + "precision": 0.8636694045420532, + "recall": 0.8535389195519456, + "f1-score": 0.8584642718288459, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.911880127139817, + "recall": 0.9129794415464867, + "f1-score": 0.9123649127866912, + "support": 16295.0 + }, + "cross-entropy": 0.19078130335337062, + "roc-auc": 0.9593891219693567, + "score": 0.9593891219693567 + } + }, + "time": "0:03:07.006918", + "diversity": { + "train": { + "mean_centered_corr": 0.08914346992969513, + "mean_pairwise_disagreement": 0.03838632562640928, + "member_std": 0.037065718322992325 + }, + "val": { + "mean_centered_corr": 0.09447728097438812, + "mean_pairwise_disagreement": 0.0372919796949436, + "member_std": 0.04158519208431244 + }, + "test": { + "mean_centered_corr": 0.11072365194559097, + "mean_pairwise_disagreement": 0.045909714042502645, + "member_std": 0.04358496516942978 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31 + ], + "head_selection_time": "0:00:00.952109" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-best-head-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-best-head-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-best-head-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-best-head-evaluation/1/report.json new file mode 100644 index 000000000..a9dfa7308 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-best-head-evaluation/1/report.json @@ -0,0 +1,298 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + }, + "mask_stats": { + "keep_rate_actual": 0.9529152512550354, + "min_features_per_member": 274.0, + "mean_features_per_member": 283.96875, + "max_features_per_member": 290.0, + "min_members_per_feature": 25.0, + "mean_members_per_feature": 30.493288040161133, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9106552463862567, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9437590003006694, + "recall": 0.9832061152601239, + "f1-score": 0.9630787946644848, + "support": 181971.0 + }, + "1": { + "precision": 0.9120398353625191, + "recall": 0.7482349051925665, + "f1-score": 0.8220567633476885, + "support": 42349.0 + }, + "accuracy": 0.9388462910128388, + "macro avg": { + "precision": 0.9278994178315942, + "recall": 0.8657205102263452, + "f1-score": 0.8925677790060866, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9377707918664427, + "recall": 0.9388462910128388, + "f1-score": 0.9364554797383303, + "support": 224320.0 + }, + "cross-entropy": 0.14120537728706906, + "roc-auc": 0.9768407671782054, + "score": 0.9768407671782054 + }, + "val": { + "0": { + "precision": 0.9325459913695208, + "recall": 0.9810644525416642, + "f1-score": 0.9561901435100283, + "support": 16741.0 + }, + "1": { + "precision": 0.8745051464766429, + "recall": 0.6502796585222255, + "f1-score": 0.7459057909842985, + "support": 3397.0 + }, + "accuracy": 0.9252656668984011, + "macro avg": { + "precision": 0.9035255689230819, + "recall": 0.8156720555319448, + "f1-score": 0.8510479672471634, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9227553095688898, + "recall": 0.9252656668984011, + "f1-score": 0.9207181033109071, + "support": 20138.0 + }, + "cross-entropy": 0.18368600862444018, + "roc-auc": 0.9550071913296724, + "score": 0.9550071913296724 + }, + "test": { + "0": { + "precision": 0.9260352898811667, + "recall": 0.9780177987373545, + "f1-score": 0.951316957679787, + "support": 13147.0 + }, + "1": { + "precision": 0.8800829875518672, + "recall": 0.6737611181702668, + "f1-score": 0.7632241813602015, + "support": 3148.0 + }, + "accuracy": 0.9192390303774164, + "macro avg": { + "precision": 0.903059138716517, + "recall": 0.8258894584538107, + "f1-score": 0.8572705695199943, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9171578521559359, + "recall": 0.9192390303774164, + "f1-score": 0.9149796726319774, + "support": 16295.0 + }, + "cross-entropy": 0.18586409414781727, + "roc-auc": 0.9596843251014889, + "score": 0.9596843251014889 + } + }, + "time": "0:03:09.746628", + "diversity": { + "train": { + "mean_centered_corr": 0.08831919729709625, + "mean_pairwise_disagreement": 0.041946592713174725, + "member_std": 0.03971056640148163 + }, + "val": { + "mean_centered_corr": 0.10093759745359421, + "mean_pairwise_disagreement": 0.040990251939039976, + "member_std": 0.044565752148628235 + }, + "test": { + "mean_centered_corr": 0.11717803031206131, + "mean_pairwise_disagreement": 0.04949915370834117, + "member_std": 0.04652492329478264 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 11 + ], + "head_selection_time": "0:00:00.926202" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-best-head-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-best-head-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-best-head-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-best-head-evaluation/2/report.json new file mode 100644 index 000000000..1e68d7660 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-best-head-evaluation/2/report.json @@ -0,0 +1,298 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + }, + "mask_stats": { + "keep_rate_actual": 0.9485108852386475, + "min_features_per_member": 274.0, + "mean_features_per_member": 282.65625, + "max_features_per_member": 289.0, + "min_members_per_feature": 25.0, + "mean_members_per_feature": 30.35234832763672, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9021152843018349, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9364448961509395, + "recall": 0.9873606233960356, + "f1-score": 0.9612289881124343, + "support": 181971.0 + }, + "1": { + "precision": 0.9291326452010477, + "recall": 0.7120593166308532, + "f1-score": 0.8062403080049195, + "support": 42349.0 + }, + "accuracy": 0.9353869472182597, + "macro avg": { + "precision": 0.9327887706759936, + "recall": 0.8497099700134444, + "f1-score": 0.883734648058677, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9350644284464237, + "recall": 0.9353869472182597, + "f1-score": 0.931968932772415, + "support": 224320.0 + }, + "cross-entropy": 0.14661004133947955, + "roc-auc": 0.9751240053891049, + "score": 0.9751240053891049 + }, + "val": { + "0": { + "precision": 0.9273278670111198, + "recall": 0.9863210083029688, + "f1-score": 0.9559151301126002, + "support": 16741.0 + }, + "1": { + "precision": 0.9018010291595198, + "recall": 0.6190756549896967, + "f1-score": 0.7341595391865945, + "support": 3397.0 + }, + "accuracy": 0.924371834343033, + "macro avg": { + "precision": 0.9145644480853198, + "recall": 0.8026983316463328, + "f1-score": 0.8450373346495974, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9230218451528475, + "recall": 0.924371834343033, + "f1-score": 0.9185080518339408, + "support": 20138.0 + }, + "cross-entropy": 0.18460242926495607, + "roc-auc": 0.9545391258255768, + "score": 0.9545391258255768 + }, + "test": { + "0": { + "precision": 0.9253505933117584, + "recall": 0.9787023655586826, + "f1-score": 0.9512790181871951, + "support": 13147.0 + }, + "1": { + "precision": 0.8828451882845189, + "recall": 0.670266836086404, + "f1-score": 0.7620079451065367, + "support": 3148.0 + }, + "accuracy": 0.9191162933415158, + "macro avg": { + "precision": 0.9040978907981386, + "recall": 0.8244846008225433, + "f1-score": 0.8566434816468659, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9171390551082756, + "recall": 0.9191162933415158, + "f1-score": 0.9147141002333495, + "support": 16295.0 + }, + "cross-entropy": 0.18885611976418637, + "roc-auc": 0.9588200872762291, + "score": 0.9588200872762291 + } + }, + "time": "0:03:06.503534", + "diversity": { + "train": { + "mean_centered_corr": 0.08432834595441818, + "mean_pairwise_disagreement": 0.03995795716660531, + "member_std": 0.04036388918757439 + }, + "val": { + "mean_centered_corr": 0.0944766253232956, + "mean_pairwise_disagreement": 0.03774069805439243, + "member_std": 0.043501291424036026 + }, + "test": { + "mean_centered_corr": 0.10949686914682388, + "mean_pairwise_disagreement": 0.04558554969365232, + "member_std": 0.04584404453635216 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 16 + ], + "head_selection_time": "0:00:01.499174" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/0.toml new file mode 100644 index 000000000..49f80d6e4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/0.toml @@ -0,0 +1,47 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.95 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/0/report.json new file mode 100644 index 000000000..24e827ce9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/0/report.json @@ -0,0 +1,294 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9506081938743591, + "min_features_per_member": 275.0, + "mean_features_per_member": 283.28125, + "max_features_per_member": 292.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.419462203979492, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9056525652610176, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9496583046818625, + "recall": 0.9835852965582428, + "f1-score": 0.9663241038432363, + "support": 181971.0 + }, + "1": { + "precision": 0.9166759651863423, + "recall": 0.7759569293253678, + "f1-score": 0.8404670255892169, + "support": 42349.0 + }, + "accuracy": 0.9443874821683309, + "macro avg": { + "precision": 0.9331671349341024, + "recall": 0.8797711129418053, + "f1-score": 0.9033955647162266, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9434316236222344, + "recall": 0.9443874821683309, + "f1-score": 0.9425637552029926, + "support": 224320.0 + }, + "cross-entropy": 0.1349038769118999, + "roc-auc": 0.9810848735528083, + "score": 0.9810848735528083 + }, + "val": { + "0": { + "precision": 0.9357175450604609, + "recall": 0.979929514365928, + "f1-score": 0.9573133370291483, + "support": 16741.0 + }, + "1": { + "precision": 0.8710667689946278, + "recall": 0.6682366794230203, + "f1-score": 0.756288522405464, + "support": 3397.0 + }, + "accuracy": 0.9273512761942596, + "macro avg": { + "precision": 0.9033921570275443, + "recall": 0.8240830968944741, + "f1-score": 0.8568009297173061, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.924811859972784, + "recall": 0.9273512761942596, + "f1-score": 0.9234032518530308, + "support": 20138.0 + }, + "cross-entropy": 0.17405502211594576, + "roc-auc": 0.9584246137411132, + "score": 0.9584246137411132 + }, + "test": { + "0": { + "precision": 0.9328956965718453, + "recall": 0.9728455160873203, + "f1-score": 0.9524518747440146, + "support": 13147.0 + }, + "1": { + "precision": 0.8618955512572534, + "recall": 0.7077509529860229, + "f1-score": 0.7772544915402059, + "support": 3148.0 + }, + "accuracy": 0.9216324025774778, + "macro avg": { + "precision": 0.8973956239145493, + "recall": 0.8402982345366716, + "f1-score": 0.8648531831421102, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9191793137887625, + "recall": 0.9216324025774778, + "f1-score": 0.9186058261201674, + "support": 16295.0 + }, + "cross-entropy": 0.1782989047843598, + "roc-auc": 0.962318960200698, + "score": 0.962318960200698 + } + }, + "time": "0:03:07.006918", + "diversity": { + "train": { + "mean_centered_corr": 0.08914346992969513, + "mean_pairwise_disagreement": 0.03838632562640928, + "member_std": 0.037065718322992325 + }, + "val": { + "mean_centered_corr": 0.09447728097438812, + "mean_pairwise_disagreement": 0.0372919796949436, + "member_std": 0.04158519208431244 + }, + "test": { + "mean_centered_corr": 0.11072365194559097, + "mean_pairwise_disagreement": 0.045909714042502645, + "member_std": 0.04358496516942978 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/1.toml new file mode 100644 index 000000000..7df284f20 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/1.toml @@ -0,0 +1,47 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 1 +head_selection = true + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.95 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 1 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/1/report.json new file mode 100644 index 000000000..f795b8ee6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/1/report.json @@ -0,0 +1,294 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + }, + "mask_stats": { + "keep_rate_actual": 0.9529152512550354, + "min_features_per_member": 274.0, + "mean_features_per_member": 283.96875, + "max_features_per_member": 290.0, + "min_members_per_feature": 25.0, + "mean_members_per_feature": 30.493288040161133, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9106552463862567, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9512478050337891, + "recall": 0.9823982942336966, + "f1-score": 0.9665721368690542, + "support": 181971.0 + }, + "1": { + "precision": 0.911981313547678, + "recall": 0.7836548678835391, + "f1-score": 0.8429621915442157, + "support": 42349.0 + }, + "accuracy": 0.9448778530670471, + "macro avg": { + "precision": 0.9316145592907336, + "recall": 0.8830265810586179, + "f1-score": 0.9047671642066349, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9438347493635622, + "recall": 0.9448778530670471, + "f1-score": 0.9432360207199744, + "support": 224320.0 + }, + "cross-entropy": 0.13516445510887865, + "roc-auc": 0.9814280207794918, + "score": 0.9814280207794918 + }, + "val": { + "0": { + "precision": 0.9362831858407079, + "recall": 0.9795711128367481, + "f1-score": 0.9574381130312938, + "support": 16741.0 + }, + "1": { + "precision": 0.8696149447197865, + "recall": 0.6714748307329997, + "f1-score": 0.7578073089700996, + "support": 3397.0 + }, + "accuracy": 0.9275995630151952, + "macro avg": { + "precision": 0.9029490652802472, + "recall": 0.825522971784874, + "f1-score": 0.8576227110006966, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.925037182509306, + "recall": 0.9275995630151952, + "f1-score": 0.9237631780131254, + "support": 20138.0 + }, + "cross-entropy": 0.17412061097796955, + "roc-auc": 0.9585282902898349, + "score": 0.9585282902898349 + }, + "test": { + "0": { + "precision": 0.9320947176684882, + "recall": 0.973073705027763, + "f1-score": 0.9521434950878238, + "support": 13147.0 + }, + "1": { + "precision": 0.8622568093385214, + "recall": 0.7039390088945362, + "f1-score": 0.7750961874781392, + "support": 3148.0 + }, + "accuracy": 0.9210800859159252, + "macro avg": { + "precision": 0.8971757635035048, + "recall": 0.8385063569611496, + "f1-score": 0.8636198412829815, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9186028652338312, + "recall": 0.9210800859159252, + "f1-score": 0.9179400630930225, + "support": 16295.0 + }, + "cross-entropy": 0.17837645706043, + "roc-auc": 0.962430602195543, + "score": 0.962430602195543 + } + }, + "time": "0:03:09.746628", + "diversity": { + "train": { + "mean_centered_corr": 0.08831919729709625, + "mean_pairwise_disagreement": 0.041946592713174725, + "member_std": 0.03971056640148163 + }, + "val": { + "mean_centered_corr": 0.10093759745359421, + "mean_pairwise_disagreement": 0.040990251939039976, + "member_std": 0.044565752148628235 + }, + "test": { + "mean_centered_corr": 0.11717803031206131, + "mean_pairwise_disagreement": 0.04949915370834117, + "member_std": 0.04652492329478264 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/2.toml new file mode 100644 index 000000000..22936b0f3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/2.toml @@ -0,0 +1,47 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 2 +head_selection = true + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.95 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 2 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/2/report.json new file mode 100644 index 000000000..f496b64bd --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/2/report.json @@ -0,0 +1,294 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + }, + "mask_stats": { + "keep_rate_actual": 0.9485108852386475, + "min_features_per_member": 274.0, + "mean_features_per_member": 282.65625, + "max_features_per_member": 289.0, + "min_members_per_feature": 25.0, + "mean_members_per_feature": 30.35234832763672, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9021152843018349, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9471737091181214, + "recall": 0.987684850882833, + "f1-score": 0.9670051785594189, + "support": 181971.0 + }, + "1": { + "precision": 0.9351675056413817, + "recall": 0.7633001959904602, + "f1-score": 0.8405382565169343, + "support": 42349.0 + }, + "accuracy": 0.9453236447931527, + "macro avg": { + "precision": 0.9411706073797516, + "recall": 0.8754925234366466, + "f1-score": 0.9037717175381765, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9449070779169959, + "recall": 0.9453236447931527, + "f1-score": 0.9431296985238573, + "support": 224320.0 + }, + "cross-entropy": 0.13396425577185272, + "roc-auc": 0.9820352766151116, + "score": 0.9820352766151116 + }, + "val": { + "0": { + "precision": 0.9312351678155724, + "recall": 0.9844692670688728, + "f1-score": 0.9571125758587647, + "support": 16741.0 + }, + "1": { + "precision": 0.8934426229508197, + "recall": 0.6417427141595525, + "f1-score": 0.7469590543087202, + "support": 3397.0 + }, + "accuracy": 0.9266560730956401, + "macro avg": { + "precision": 0.912338895383196, + "recall": 0.8131059906142126, + "f1-score": 0.8520358150837425, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9248600920927813, + "recall": 0.9266560730956401, + "f1-score": 0.9216626050222616, + "support": 20138.0 + }, + "cross-entropy": 0.17636787506935708, + "roc-auc": 0.9583243485306635, + "score": 0.9583243485306635 + }, + "test": { + "0": { + "precision": 0.9282006920415224, + "recall": 0.9793869323800106, + "f1-score": 0.9531070728006218, + "support": 13147.0 + }, + "1": { + "precision": 0.8881551795295088, + "recall": 0.6836086404066074, + "f1-score": 0.7725722491473703, + "support": 3148.0 + }, + "accuracy": 0.9222460877569807, + "macro avg": { + "precision": 0.9081779357855156, + "recall": 0.8314977863933091, + "f1-score": 0.8628396609739961, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9204643757857496, + "recall": 0.9222460877569807, + "f1-score": 0.9182298942267995, + "support": 16295.0 + }, + "cross-entropy": 0.1797688037922231, + "roc-auc": 0.9625461826483815, + "score": 0.9625461826483815 + } + }, + "time": "0:03:06.503534", + "diversity": { + "train": { + "mean_centered_corr": 0.08432834595441818, + "mean_pairwise_disagreement": 0.03995795716660531, + "member_std": 0.04036388918757439 + }, + "val": { + "mean_centered_corr": 0.0944766253232956, + "mean_pairwise_disagreement": 0.03774069805439243, + "member_std": 0.043501291424036026 + }, + "test": { + "mean_centered_corr": 0.10949686914682388, + "mean_pairwise_disagreement": 0.04558554969365232, + "member_std": 0.04584404453635216 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..633179f9a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,304 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9506081938743591, + "min_features_per_member": 275.0, + "mean_features_per_member": 283.28125, + "max_features_per_member": 292.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.419462203979492, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9056525652610176, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9503279377423628, + "recall": 0.9817773161657627, + "f1-score": 0.9657966721086376, + "support": 181971.0 + }, + "1": { + "precision": 0.9087180334186693, + "recall": 0.779498925594465, + "f1-score": 0.8391631501347298, + "support": 42349.0 + }, + "accuracy": 0.943589514978602, + "macro avg": { + "precision": 0.929522985580516, + "recall": 0.8806381208801138, + "f1-score": 0.9024799111216837, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9424724730570735, + "recall": 0.943589514978602, + "f1-score": 0.941889739948897, + "support": 224320.0 + }, + "cross-entropy": 0.13479785159389124, + "roc-auc": 0.9805680231536485, + "score": 0.9805680231536485 + }, + "val": { + "0": { + "precision": 0.9373999542648068, + "recall": 0.9794516456603548, + "f1-score": 0.9579645371425233, + "support": 16741.0 + }, + "1": { + "precision": 0.8699924414210128, + "recall": 0.6776567559611422, + "f1-score": 0.7618732417673341, + "support": 3397.0 + }, + "accuracy": 0.9285430529347503, + "macro avg": { + "precision": 0.9036961978429099, + "recall": 0.8285542008107485, + "f1-score": 0.8599188894549287, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9260292460946623, + "recall": 0.9285430529347503, + "f1-score": 0.9248866679206782, + "support": 20138.0 + }, + "cross-entropy": 0.17265539939712965, + "roc-auc": 0.9592957095194116, + "score": 0.9592957095194116 + }, + "test": { + "0": { + "precision": 0.9348464815630968, + "recall": 0.9680535483380239, + "f1-score": 0.9511602705429543, + "support": 13147.0 + }, + "1": { + "precision": 0.8433420365535248, + "recall": 0.7182337992376112, + "f1-score": 0.7757762909589981, + "support": 3148.0 + }, + "accuracy": 0.919791347038969, + "macro avg": { + "precision": 0.8890942590583109, + "recall": 0.8431436737878175, + "f1-score": 0.8634682807509761, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9171689121927296, + "recall": 0.919791347038969, + "f1-score": 0.9172781737199843, + "support": 16295.0 + }, + "cross-entropy": 0.1790101347961027, + "roc-auc": 0.9620218603265258, + "score": 0.9620218603265258 + } + }, + "time": "0:03:07.006918", + "diversity": { + "train": { + "mean_centered_corr": 0.08914346992969513, + "mean_pairwise_disagreement": 0.03838632562640928, + "member_std": 0.037065718322992325 + }, + "val": { + "mean_centered_corr": 0.09447728097438812, + "mean_pairwise_disagreement": 0.0372919796949436, + "member_std": 0.04158519208431244 + }, + "test": { + "mean_centered_corr": 0.11072365194559097, + "mean_pairwise_disagreement": 0.045909714042502645, + "member_std": 0.04358496516942978 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31, + 5, + 28, + 20, + 9, + 6, + 23 + ], + "head_selection_time": "0:00:08.820400" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/1/report.json new file mode 100644 index 000000000..b8a656ee7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/1/report.json @@ -0,0 +1,310 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + }, + "mask_stats": { + "keep_rate_actual": 0.9529152512550354, + "min_features_per_member": 274.0, + "mean_features_per_member": 283.96875, + "max_features_per_member": 290.0, + "min_members_per_feature": 25.0, + "mean_members_per_feature": 30.493288040161133, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9106552463862567, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9512209410699545, + "recall": 0.9815080424902869, + "f1-score": 0.9661271826384231, + "support": 181971.0 + }, + "1": { + "precision": 0.9079469292846396, + "recall": 0.7837257078089211, + "f1-score": 0.8412754739937138, + "support": 42349.0 + }, + "accuracy": 0.9441690442225392, + "macro avg": { + "precision": 0.929583935177297, + "recall": 0.882616875149604, + "f1-score": 0.9037013283160684, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9430513123025851, + "recall": 0.9441690442225392, + "f1-score": 0.9425566360558857, + "support": 224320.0 + }, + "cross-entropy": 0.13677446177980773, + "roc-auc": 0.9808759836686648, + "score": 0.9808759836686648 + }, + "val": { + "0": { + "precision": 0.9363148275074252, + "recall": 0.9792127113075683, + "f1-score": 0.9572834243335572, + "support": 16741.0 + }, + "1": { + "precision": 0.867680608365019, + "recall": 0.671769208124816, + "f1-score": 0.7572590011614402, + "support": 3397.0 + }, + "accuracy": 0.9273512761942596, + "macro avg": { + "precision": 0.9019977179362222, + "recall": 0.8254909597161921, + "f1-score": 0.8572712127474986, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9247371910774543, + "recall": 0.9273512761942596, + "f1-score": 0.9235420912560082, + "support": 20138.0 + }, + "cross-entropy": 0.1738401509177298, + "roc-auc": 0.9591725848256957, + "score": 0.9591725848256957 + }, + "test": { + "0": { + "precision": 0.9327056419239471, + "recall": 0.9720088233056972, + "f1-score": 0.9519517282479142, + "support": 13147.0 + }, + "1": { + "precision": 0.8581341557440246, + "recall": 0.7071156289707751, + "f1-score": 0.7753396029258098, + "support": 3148.0 + }, + "accuracy": 0.920834611844124, + "macro avg": { + "precision": 0.8954198988339859, + "recall": 0.8395622261382361, + "f1-score": 0.8636456655868621, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9182993186042542, + "recall": 0.920834611844124, + "f1-score": 0.9178323682900138, + "support": 16295.0 + }, + "cross-entropy": 0.17951798610400288, + "roc-auc": 0.9622278199335073, + "score": 0.9622278199335073 + } + }, + "time": "0:03:09.746628", + "diversity": { + "train": { + "mean_centered_corr": 0.08831919729709625, + "mean_pairwise_disagreement": 0.041946592713174725, + "member_std": 0.03971056640148163 + }, + "val": { + "mean_centered_corr": 0.10093759745359421, + "mean_pairwise_disagreement": 0.040990251939039976, + "member_std": 0.044565752148628235 + }, + "test": { + "mean_centered_corr": 0.11717803031206131, + "mean_pairwise_disagreement": 0.04949915370834117, + "member_std": 0.04652492329478264 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 11, + 14, + 0, + 21, + 3, + 8, + 26, + 27, + 25, + 20, + 23, + 16, + 14 + ], + "head_selection_time": "0:00:13.167718" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/2/report.json new file mode 100644 index 000000000..d33592680 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/2/report.json @@ -0,0 +1,309 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + }, + "mask_stats": { + "keep_rate_actual": 0.9485108852386475, + "min_features_per_member": 274.0, + "mean_features_per_member": 282.65625, + "max_features_per_member": 289.0, + "min_members_per_feature": 25.0, + "mean_members_per_feature": 30.35234832763672, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9021152843018349, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9466985010435772, + "recall": 0.9870748635771633, + "f1-score": 0.9664651617016812, + "support": 181971.0 + }, + "1": { + "precision": 0.9319995374118192, + "recall": 0.7611986115374625, + "f1-score": 0.8379843248372044, + "support": 42349.0 + }, + "accuracy": 0.9444320613409415, + "macro avg": { + "precision": 0.9393490192276982, + "recall": 0.8741367375573129, + "f1-score": 0.9022247432694428, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9439235081279151, + "recall": 0.9444320613409415, + "f1-score": 0.942209478033824, + "support": 224320.0 + }, + "cross-entropy": 0.13594152929391584, + "roc-auc": 0.9810912447120521, + "score": 0.9810912447120521 + }, + "val": { + "0": { + "precision": 0.9318143269067677, + "recall": 0.9844692670688728, + "f1-score": 0.9574183803880563, + "support": 16741.0 + }, + "1": { + "precision": 0.893920848633211, + "recall": 0.6449808654695319, + "f1-score": 0.7493160054719562, + "support": 3397.0 + }, + "accuracy": 0.9272023041016982, + "macro avg": { + "precision": 0.9128675877699893, + "recall": 0.8147250662692024, + "f1-score": 0.8533671929300062, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9254222251243032, + "recall": 0.9272023041016982, + "f1-score": 0.9223144094083169, + "support": 20138.0 + }, + "cross-entropy": 0.17569113158213984, + "roc-auc": 0.9590785004678369, + "score": 0.9590785004678369 + }, + "test": { + "0": { + "precision": 0.9279227331699582, + "recall": 0.9792348064197155, + "f1-score": 0.952888494134192, + "support": 13147.0 + }, + "1": { + "precision": 0.8872366790582404, + "recall": 0.6823379923761118, + "f1-score": 0.7714131801041479, + "support": 3148.0 + }, + "accuracy": 0.9218778766492789, + "macro avg": { + "precision": 0.9075797061140993, + "recall": 0.8307863993979137, + "f1-score": 0.86215083711917, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9200626719030857, + "recall": 0.9218778766492789, + "f1-score": 0.9178296240165744, + "support": 16295.0 + }, + "cross-entropy": 0.17937786610967274, + "roc-auc": 0.962897309467792, + "score": 0.962897309467792 + } + }, + "time": "0:03:06.503534", + "diversity": { + "train": { + "mean_centered_corr": 0.08432834595441818, + "mean_pairwise_disagreement": 0.03995795716660531, + "member_std": 0.04036388918757439 + }, + "val": { + "mean_centered_corr": 0.0944766253232956, + "mean_pairwise_disagreement": 0.03774069805439243, + "member_std": 0.043501291424036026 + }, + "test": { + "mean_centered_corr": 0.10949686914682388, + "mean_pairwise_disagreement": 0.04558554969365232, + "member_std": 0.04584404453635216 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 16, + 3, + 10, + 30, + 0, + 6, + 1, + 27, + 11, + 4, + 3, + 2 + ], + "head_selection_time": "0:00:16.177425" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..7721a02fc --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-best-head-evaluation/0/report.json @@ -0,0 +1,275 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9413658167129513, + "recall": 0.9835633150337142, + "f1-score": 0.9620020478421065, + "support": 181971.0 + }, + "1": { + "precision": 0.9125233972859148, + "recall": 0.7367588372806914, + "f1-score": 0.8152754732757607, + "support": 42349.0 + }, + "accuracy": 0.9369695078459344, + "macro avg": { + "precision": 0.926944606999433, + "recall": 0.8601610761572027, + "f1-score": 0.8886387605589336, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9359207042828712, + "recall": 0.9369695078459344, + "f1-score": 0.9343017816763158, + "support": 224320.0 + }, + "cross-entropy": 0.14393086493300278, + "roc-auc": 0.9759690841627111, + "score": 0.9759690841627111 + }, + "val": { + "0": { + "precision": 0.9325333937061354, + "recall": 0.9841705991278896, + "f1-score": 0.9576564270975617, + "support": 16741.0 + }, + "1": { + "precision": 0.8927125506072875, + "recall": 0.6491021489549602, + "f1-score": 0.7516618373956026, + "support": 3397.0 + }, + "accuracy": 0.9276492203793822, + "macro avg": { + "precision": 0.9126229721567114, + "recall": 0.8166363740414249, + "f1-score": 0.8546591322465822, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.925816172333269, + "recall": 0.9276492203793822, + "f1-score": 0.9229080101138715, + "support": 20138.0 + }, + "cross-entropy": 0.17925081582806562, + "roc-auc": 0.9560453459701026, + "score": 0.9560453459701026 + }, + "test": { + "0": { + "precision": 0.9250304899921085, + "recall": 0.9807560660226667, + "f1-score": 0.9520785645721037, + "support": 13147.0 + }, + "1": { + "precision": 0.8926146010186757, + "recall": 0.6680432020330368, + "f1-score": 0.764171511627907, + "support": 3148.0 + }, + "accuracy": 0.9203436637005217, + "macro avg": { + "precision": 0.9088225455053921, + "recall": 0.8243996340278518, + "f1-score": 0.8581250381000054, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9187681261695637, + "recall": 0.9203436637005217, + "f1-score": 0.9157771590692911, + "support": 16295.0 + }, + "cross-entropy": 0.18465226452577868, + "roc-auc": 0.9600924508313724, + "score": 0.9600924508313724 + } + }, + "time": "0:06:58.103811", + "diversity": { + "train": { + "mean_centered_corr": 0.05266324058175087, + "mean_pairwise_disagreement": 0.03131294111810317, + "member_std": 0.028657343238592148 + }, + "val": { + "mean_centered_corr": 0.06352649629116058, + "mean_pairwise_disagreement": 0.02856399712948398, + "member_std": 0.030144035816192627 + }, + "test": { + "mean_centered_corr": 0.0760873481631279, + "mean_pairwise_disagreement": 0.03589971691296558, + "member_std": 0.032248713076114655 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31 + ], + "head_selection_time": "0:00:00.778477" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-best-head-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-best-head-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-best-head-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-best-head-evaluation/1/report.json new file mode 100644 index 000000000..1fc960885 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-best-head-evaluation/1/report.json @@ -0,0 +1,275 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9482846874900025, + "recall": 0.9773370482109787, + "f1-score": 0.9625917075533738, + "support": 181971.0 + }, + "1": { + "precision": 0.8878555501169304, + "recall": 0.770974521240171, + "f1-score": 0.8252973218912326, + "support": 42349.0 + }, + "accuracy": 0.938378209700428, + "macro avg": { + "precision": 0.9180701188034665, + "recall": 0.8741557847255748, + "f1-score": 0.8939445147223032, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9368763710732219, + "recall": 0.938378209700428, + "f1-score": 0.9366721286553442, + "support": 224320.0 + }, + "cross-entropy": 0.1408223018750539, + "roc-auc": 0.9772865244821658, + "score": 0.9772865244821658 + }, + "val": { + "0": { + "precision": 0.9336525307797537, + "recall": 0.9784361746610118, + "f1-score": 0.9555199066647222, + "support": 16741.0 + }, + "1": { + "precision": 0.8608326908249807, + "recall": 0.6573447159258169, + "f1-score": 0.7454515105992322, + "support": 3397.0 + }, + "accuracy": 0.9242725196146588, + "macro avg": { + "precision": 0.8972426108023672, + "recall": 0.8178904452934144, + "f1-score": 0.8504857086319773, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9213688384405759, + "recall": 0.9242725196146588, + "f1-score": 0.9200842953113372, + "support": 20138.0 + }, + "cross-entropy": 0.18036337330439542, + "roc-auc": 0.9560549117846388, + "score": 0.9560549117846388 + }, + "test": { + "0": { + "precision": 0.9309239288837073, + "recall": 0.9717806343652544, + "f1-score": 0.9509136243533921, + "support": 13147.0 + }, + "1": { + "precision": 0.8556981719175418, + "recall": 0.6988564167725541, + "f1-score": 0.7693652736492393, + "support": 3148.0 + }, + "accuracy": 0.9190549248235655, + "macro avg": { + "precision": 0.8933110504006245, + "recall": 0.8353185255689042, + "f1-score": 0.8601394490013157, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9163912082375282, + "recall": 0.9190549248235655, + "f1-score": 0.9158406444198742, + "support": 16295.0 + }, + "cross-entropy": 0.18712658329229517, + "roc-auc": 0.9591263205069757, + "score": 0.9591263205069757 + } + }, + "time": "0:06:59.217282", + "diversity": { + "train": { + "mean_centered_corr": 0.04887603968381882, + "mean_pairwise_disagreement": 0.031215747736528462, + "member_std": 0.028810881078243256 + }, + "val": { + "mean_centered_corr": 0.053518619388341904, + "mean_pairwise_disagreement": 0.027906437516619198, + "member_std": 0.030356688424944878 + }, + "test": { + "mean_centered_corr": 0.06887096166610718, + "mean_pairwise_disagreement": 0.035209073632323395, + "member_std": 0.032625459134578705 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 14 + ], + "head_selection_time": "0:00:00.940553" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-best-head-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-best-head-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-best-head-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-best-head-evaluation/2/report.json new file mode 100644 index 000000000..e1d98b16e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-best-head-evaluation/2/report.json @@ -0,0 +1,275 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.942254416998281, + "recall": 0.9850250864148683, + "f1-score": 0.9631651630028856, + "support": 181971.0 + }, + "1": { + "precision": 0.920062190149315, + "recall": 0.7406078065597771, + "f1-score": 0.8206389492137418, + "support": 42349.0 + }, + "accuracy": 0.9388819543509273, + "macro avg": { + "precision": 0.931158303573798, + "recall": 0.8628164464873227, + "f1-score": 0.8919020561083137, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9380647833729829, + "recall": 0.9388819543509273, + "f1-score": 0.9362578759675947, + "support": 224320.0 + }, + "cross-entropy": 0.13956166211953258, + "roc-auc": 0.9780078991108505, + "score": 0.9780078991108505 + }, + "val": { + "0": { + "precision": 0.9299007444168734, + "recall": 0.9849471357744459, + "f1-score": 0.9566327270617585, + "support": 16741.0 + }, + "1": { + "precision": 0.8952618453865336, + "recall": 0.6340889019723285, + "f1-score": 0.7423746338100983, + "support": 3397.0 + }, + "accuracy": 0.9257622405402721, + "macro avg": { + "precision": 0.9125812949017036, + "recall": 0.8095180188733873, + "f1-score": 0.8495036804359284, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9240576448038997, + "recall": 0.9257622405402721, + "f1-score": 0.9204903721717053, + "support": 20138.0 + }, + "cross-entropy": 0.19004422805681795, + "roc-auc": 0.9556372338569274, + "score": 0.9556372338569274 + }, + "test": { + "0": { + "precision": 0.9235277301315037, + "recall": 0.9828858294667985, + "f1-score": 0.9522826928037142, + "support": 13147.0 + }, + "1": { + "precision": 0.9023013460703431, + "recall": 0.6601016518424396, + "f1-score": 0.7624289121262153, + "support": 3148.0 + }, + "accuracy": 0.9205277692543725, + "macro avg": { + "precision": 0.9129145381009234, + "recall": 0.821493740654619, + "f1-score": 0.8573558024649648, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9194270454414434, + "recall": 0.9205277692543725, + "f1-score": 0.9156052026795801, + "support": 16295.0 + }, + "cross-entropy": 0.19223543365810944, + "roc-auc": 0.9608528752531366, + "score": 0.9608528752531366 + } + }, + "time": "0:07:13.032423", + "diversity": { + "train": { + "mean_centered_corr": 0.05698179081082344, + "mean_pairwise_disagreement": 0.03173294702843864, + "member_std": 0.031025612726807594 + }, + "val": { + "mean_centered_corr": 0.06959165632724762, + "mean_pairwise_disagreement": 0.030371885602247718, + "member_std": 0.033528175204992294 + }, + "test": { + "mean_centered_corr": 0.08432681858539581, + "mean_pairwise_disagreement": 0.03709145393896802, + "member_std": 0.03507966548204422 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 1 + ], + "head_selection_time": "0:00:01.404215" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-evaluation/0.toml new file mode 100644 index 000000000..601f75de8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-evaluation/0.toml @@ -0,0 +1,41 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-evaluation/0/report.json new file mode 100644 index 000000000..4bb9206ba --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-evaluation/0/report.json @@ -0,0 +1,271 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9463105683240804, + "recall": 0.9844810436827847, + "f1-score": 0.9650185034555944, + "support": 181971.0 + }, + "1": { + "precision": 0.9193350281356223, + "recall": 0.7599943328059694, + "f1-score": 0.8321052767651698, + "support": 42349.0 + }, + "accuracy": 0.9421005706134095, + "macro avg": { + "precision": 0.9328227982298514, + "recall": 0.872237688244377, + "f1-score": 0.898561890110382, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9412179009228634, + "recall": 0.9421005706134095, + "f1-score": 0.9399260362787364, + "support": 224320.0 + }, + "cross-entropy": 0.1363573248497679, + "roc-auc": 0.9795959593437451, + "score": 0.9795959593437451 + }, + "val": { + "0": { + "precision": 0.9344513928368391, + "recall": 0.9818409891882205, + "f1-score": 0.9575602225393959, + "support": 16741.0 + }, + "1": { + "precision": 0.880690737833595, + "recall": 0.6605828672357963, + "f1-score": 0.7549201009251472, + "support": 3397.0 + }, + "accuracy": 0.9276492203793822, + "macro avg": { + "precision": 0.907571065335217, + "recall": 0.8212119282120084, + "f1-score": 0.8562401617322715, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9253827194310382, + "recall": 0.9276492203793822, + "f1-score": 0.9233776575814258, + "support": 20138.0 + }, + "cross-entropy": 0.17415890290359384, + "roc-auc": 0.9581626616470993, + "score": 0.9581626616470993 + }, + "test": { + "0": { + "precision": 0.9295326291419477, + "recall": 0.9772571689358789, + "f1-score": 0.9527976565686529, + "support": 13147.0 + }, + "1": { + "precision": 0.879094217549535, + "recall": 0.6905972045743329, + "f1-score": 0.7735278420209927, + "support": 3148.0 + }, + "accuracy": 0.9218778766492789, + "macro avg": { + "precision": 0.9043134233457413, + "recall": 0.8339271867551059, + "f1-score": 0.8631627492948228, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9197885285164236, + "recall": 0.9218778766492789, + "f1-score": 0.9181648626934744, + "support": 16295.0 + }, + "cross-entropy": 0.17842812985458048, + "roc-auc": 0.9620964494052155, + "score": 0.9620964494052155 + } + }, + "time": "0:06:58.103811", + "diversity": { + "train": { + "mean_centered_corr": 0.05266324058175087, + "mean_pairwise_disagreement": 0.03131294111810317, + "member_std": 0.028657343238592148 + }, + "val": { + "mean_centered_corr": 0.06352649629116058, + "mean_pairwise_disagreement": 0.02856399712948398, + "member_std": 0.030144035816192627 + }, + "test": { + "mean_centered_corr": 0.0760873481631279, + "mean_pairwise_disagreement": 0.03589971691296558, + "member_std": 0.032248713076114655 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-evaluation/1.toml new file mode 100644 index 000000000..5643d56c1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-evaluation/1.toml @@ -0,0 +1,41 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 1 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-evaluation/1/report.json new file mode 100644 index 000000000..49f122420 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-evaluation/1/report.json @@ -0,0 +1,271 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9447127920145382, + "recall": 0.9855911106714806, + "f1-score": 0.964719108376186, + "support": 181971.0 + }, + "1": { + "precision": 0.9239448875997099, + "recall": 0.7521547143970342, + "f1-score": 0.8292460689367905, + "support": 42349.0 + }, + "accuracy": 0.9415210413694722, + "macro avg": { + "precision": 0.9343288398071241, + "recall": 0.8688729125342574, + "f1-score": 0.8969825886564883, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9407920538544831, + "recall": 0.9415210413694722, + "f1-score": 0.9391433784046322, + "support": 224320.0 + }, + "cross-entropy": 0.13643410406106432, + "roc-auc": 0.9796283338071923, + "score": 0.9796283338071923 + }, + "val": { + "0": { + "precision": 0.9313542550183772, + "recall": 0.9838719311869064, + "f1-score": 0.9568930459536397, + "support": 16741.0 + }, + "1": { + "precision": 0.889930697105585, + "recall": 0.6426258463350015, + "f1-score": 0.7463247863247863, + "support": 3397.0 + }, + "accuracy": 0.9263084715463303, + "macro avg": { + "precision": 0.9106424760619811, + "recall": 0.8132488887609539, + "f1-score": 0.851608916139213, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9243666779883963, + "recall": 0.9263084715463303, + "f1-score": 0.9213731145821422, + "support": 20138.0 + }, + "cross-entropy": 0.17370455502811047, + "roc-auc": 0.9583407546059616, + "score": 0.9583407546059616 + }, + "test": { + "0": { + "precision": 0.9288657558013446, + "recall": 0.9773332319160265, + "f1-score": 0.9524833209785026, + "support": 13147.0 + }, + "1": { + "precision": 0.8789601949634444, + "recall": 0.6874205844980941, + "f1-score": 0.7714795008912656, + "support": 3148.0 + }, + "accuracy": 0.9213255599877263, + "macro avg": { + "precision": 0.9039129753823945, + "recall": 0.8323769082070602, + "f1-score": 0.8619814109348841, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9192245955977417, + "recall": 0.9213255599877263, + "f1-score": 0.9175155378772678, + "support": 16295.0 + }, + "cross-entropy": 0.17839301052302864, + "roc-auc": 0.9620763221935056, + "score": 0.9620763221935056 + } + }, + "time": "0:06:59.217282", + "diversity": { + "train": { + "mean_centered_corr": 0.04887603968381882, + "mean_pairwise_disagreement": 0.031215747736528462, + "member_std": 0.028810881078243256 + }, + "val": { + "mean_centered_corr": 0.053518619388341904, + "mean_pairwise_disagreement": 0.027906437516619198, + "member_std": 0.030356688424944878 + }, + "test": { + "mean_centered_corr": 0.06887096166610718, + "mean_pairwise_disagreement": 0.035209073632323395, + "member_std": 0.032625459134578705 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-evaluation/2.toml new file mode 100644 index 000000000..0fdbfa723 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-evaluation/2.toml @@ -0,0 +1,41 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 2 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-evaluation/2/report.json new file mode 100644 index 000000000..58043ba69 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-evaluation/2/report.json @@ -0,0 +1,271 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9463349547713352, + "recall": 0.9899929109583395, + "f1-score": 0.9676717597230466, + "support": 181971.0 + }, + "1": { + "precision": 0.946368616363315, + "recall": 0.7587664407660157, + "f1-score": 0.8422473559362017, + "support": 42349.0 + }, + "accuracy": 0.9463400499286734, + "macro avg": { + "precision": 0.9463517855673251, + "recall": 0.8743796758621776, + "f1-score": 0.9049595578296241, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9463413096873425, + "recall": 0.9463400499286734, + "f1-score": 0.943993094976394, + "support": 224320.0 + }, + "cross-entropy": 0.12561167365428139, + "roc-auc": 0.9837216635411232, + "score": 0.9837216635411232 + }, + "val": { + "0": { + "precision": 0.9312468222134342, + "recall": 0.9846484678334627, + "f1-score": 0.9572034144358632, + "support": 16741.0 + }, + "1": { + "precision": 0.8945424702503078, + "recall": 0.6417427141595525, + "f1-score": 0.747343160781625, + "support": 3397.0 + }, + "accuracy": 0.9268050451882014, + "macro avg": { + "precision": 0.912894646231871, + "recall": 0.8131955909965076, + "f1-score": 0.8522732876087441, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9250553094704239, + "recall": 0.9268050451882014, + "f1-score": 0.9218029138070297, + "support": 20138.0 + }, + "cross-entropy": 0.17572658501355687, + "roc-auc": 0.9581674973070209, + "score": 0.9581674973070209 + }, + "test": { + "0": { + "precision": 0.9270114942528735, + "recall": 0.9815166958241424, + "f1-score": 0.9534857945099199, + "support": 13147.0 + }, + "1": { + "precision": 0.8976842105263158, + "recall": 0.6772554002541296, + "f1-score": 0.7720441788882854, + "support": 3148.0 + }, + "accuracy": 0.922737035900583, + "macro avg": { + "precision": 0.9123478523895947, + "recall": 0.829386048039136, + "f1-score": 0.8627649866991026, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9213458121926585, + "recall": 0.922737035900583, + "f1-score": 0.9184334345236109, + "support": 16295.0 + }, + "cross-entropy": 0.17887720986431865, + "roc-auc": 0.9627480104988175, + "score": 0.9627480104988175 + } + }, + "time": "0:07:13.032423", + "diversity": { + "train": { + "mean_centered_corr": 0.05698179081082344, + "mean_pairwise_disagreement": 0.03173294702843864, + "member_std": 0.031025612726807594 + }, + "val": { + "mean_centered_corr": 0.06959165632724762, + "mean_pairwise_disagreement": 0.030371885602247718, + "member_std": 0.033528175204992294 + }, + "test": { + "mean_centered_corr": 0.08432681858539581, + "mean_pairwise_disagreement": 0.03709145393896802, + "member_std": 0.03507966548204422 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..b14bbaa40 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,284 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9441162844567987, + "recall": 0.985503184573366, + "f1-score": 0.9643658968753949, + "support": 181971.0 + }, + "1": { + "precision": 0.923251483765856, + "recall": 0.749344730690217, + "f1-score": 0.8272572046766856, + "support": 42349.0 + }, + "accuracy": 0.9409192225392297, + "macro avg": { + "precision": 0.9336838841113273, + "recall": 0.8674239576317915, + "f1-score": 0.8958115507760402, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9401772534098091, + "recall": 0.9409192225392297, + "f1-score": 0.9384813747377204, + "support": 224320.0 + }, + "cross-entropy": 0.13816220002148916, + "roc-auc": 0.9787552331834622, + "score": 0.9787552331834622 + }, + "val": { + "0": { + "precision": 0.931622965641953, + "recall": 0.984767935009856, + "f1-score": 0.9574585474925226, + "support": 16741.0 + }, + "1": { + "precision": 0.8955773955773956, + "recall": 0.6438033559022667, + "f1-score": 0.7491008734372324, + "support": 3397.0 + }, + "accuracy": 0.9272519614658854, + "macro avg": { + "precision": 0.9136001806096743, + "recall": 0.8142856454560614, + "f1-score": 0.8532797104648775, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.925542580225859, + "recall": 0.9272519614658854, + "f1-score": 0.9223115111053034, + "support": 20138.0 + }, + "cross-entropy": 0.17350924991795408, + "roc-auc": 0.9591329763748824, + "score": 0.9591329763748824 + }, + "test": { + "0": { + "precision": 0.9269169903610991, + "recall": 0.9801475621814862, + "f1-score": 0.9527893822322452, + "support": 13147.0 + }, + "1": { + "precision": 0.8909318846636022, + "recall": 0.6772554002541296, + "f1-score": 0.769536184804187, + "support": 3148.0 + }, + "accuracy": 0.9216324025774778, + "macro avg": { + "precision": 0.9089244375123506, + "recall": 0.828701481217808, + "f1-score": 0.8611627835182161, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9199650963607481, + "recall": 0.9216324025774778, + "f1-score": 0.9173870462087087, + "support": 16295.0 + }, + "cross-entropy": 0.17845179542990083, + "roc-auc": 0.962370812537228, + "score": 0.962370812537228 + } + }, + "time": "0:06:58.103811", + "diversity": { + "train": { + "mean_centered_corr": 0.05266324058175087, + "mean_pairwise_disagreement": 0.03131294111810317, + "member_std": 0.028657343238592148 + }, + "val": { + "mean_centered_corr": 0.06352649629116058, + "mean_pairwise_disagreement": 0.02856399712948398, + "member_std": 0.030144035816192627 + }, + "test": { + "mean_centered_corr": 0.0760873481631279, + "mean_pairwise_disagreement": 0.03589971691296558, + "member_std": 0.032248713076114655 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31, + 24, + 30, + 13, + 17, + 21, + 28, + 24, + 6, + 8 + ], + "head_selection_time": "0:00:10.755431" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/1/report.json new file mode 100644 index 000000000..24ba88a83 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/1/report.json @@ -0,0 +1,284 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9462082666286251, + "recall": 0.984145825433723, + "f1-score": 0.9648042495649691, + "support": 181971.0 + }, + "1": { + "precision": 0.917696060251619, + "recall": 0.7595929065621384, + "f1-score": 0.8311929924291362, + "support": 42349.0 + }, + "accuracy": 0.941752853067047, + "macro avg": { + "precision": 0.931952163440122, + "recall": 0.8718693659979306, + "f1-score": 0.8979986209970527, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9408254945714754, + "recall": 0.941752853067047, + "f1-score": 0.9395800023803874, + "support": 224320.0 + }, + "cross-entropy": 0.13658565047383905, + "roc-auc": 0.9794746002312951, + "score": 0.9794746002312951 + }, + "val": { + "0": { + "precision": 0.9323214589114799, + "recall": 0.9833343288931367, + "f1-score": 0.9571486714343858, + "support": 16741.0 + }, + "1": { + "precision": 0.8875453446191052, + "recall": 0.6482190167795113, + "f1-score": 0.7492344334807758, + "support": 3397.0 + }, + "accuracy": 0.9268050451882014, + "macro avg": { + "precision": 0.9099334017652925, + "recall": 0.815776672836324, + "f1-score": 0.8531915524575808, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9247683523343027, + "recall": 0.9268050451882014, + "f1-score": 0.9220764365387452, + "support": 20138.0 + }, + "cross-entropy": 0.17285360392280882, + "roc-auc": 0.9590165424760763, + "score": 0.9590165424760763 + }, + "test": { + "0": { + "precision": 0.9310144506571781, + "recall": 0.9752034684718948, + "f1-score": 0.952596775391931, + "support": 13147.0 + }, + "1": { + "precision": 0.8708399366085579, + "recall": 0.6982210927573063, + "f1-score": 0.7750352609308886, + "support": 3148.0 + }, + "accuracy": 0.921693771095428, + "macro avg": { + "precision": 0.900927193632868, + "recall": 0.8367122806146006, + "f1-score": 0.8638160181614098, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9193894509502093, + "recall": 0.921693771095428, + "f1-score": 0.9182940047553332, + "support": 16295.0 + }, + "cross-entropy": 0.17828631663015132, + "roc-auc": 0.9620755973239362, + "score": 0.9620755973239362 + } + }, + "time": "0:06:59.217282", + "diversity": { + "train": { + "mean_centered_corr": 0.04887603968381882, + "mean_pairwise_disagreement": 0.031215747736528462, + "member_std": 0.028810881078243256 + }, + "val": { + "mean_centered_corr": 0.053518619388341904, + "mean_pairwise_disagreement": 0.027906437516619198, + "member_std": 0.030356688424944878 + }, + "test": { + "mean_centered_corr": 0.06887096166610718, + "mean_pairwise_disagreement": 0.035209073632323395, + "member_std": 0.032625459134578705 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 14, + 26, + 5, + 21, + 13, + 24, + 18, + 30, + 28, + 16 + ], + "head_selection_time": "0:00:10.855848" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/2/report.json new file mode 100644 index 000000000..ddb74823e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/2/report.json @@ -0,0 +1,293 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9461253448955459, + "recall": 0.9893004929356876, + "f1-score": 0.9672313470124595, + "support": 181971.0 + }, + "1": { + "precision": 0.9428109854604201, + "recall": 0.7579399749698931, + "f1-score": 0.840327774432547, + "support": 42349.0 + }, + "accuracy": 0.9456223252496434, + "macro avg": { + "precision": 0.9444681651779829, + "recall": 0.8736202339527903, + "f1-score": 0.9037795607225032, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9454996324859608, + "recall": 0.9456223252496434, + "f1-score": 0.9432734324476115, + "support": 224320.0 + }, + "cross-entropy": 0.12704980324137066, + "roc-auc": 0.9832599119647002, + "score": 0.9832599119647002 + }, + "val": { + "0": { + "precision": 0.9311162554344757, + "recall": 0.9850666029508393, + "f1-score": 0.9573319400905608, + "support": 16741.0 + }, + "1": { + "precision": 0.8969921714050267, + "recall": 0.6408595819841036, + "f1-score": 0.7475961538461539, + "support": 3397.0 + }, + "accuracy": 0.9270036746449498, + "macro avg": { + "precision": 0.9140542134197512, + "recall": 0.8129630924674714, + "f1-score": 0.8524640469683573, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9253599979387941, + "recall": 0.9270036746449498, + "f1-score": 0.921952435379455, + "support": 20138.0 + }, + "cross-entropy": 0.17486402172446866, + "roc-auc": 0.9587859694188998, + "score": 0.9587859694188998 + }, + "test": { + "0": { + "precision": 0.9278120959632237, + "recall": 0.9825055145660607, + "f1-score": 0.9543758542982748, + "support": 13147.0 + }, + "1": { + "precision": 0.9030762747576907, + "recall": 0.6807496823379924, + "f1-score": 0.7763086397391776, + "support": 3148.0 + }, + "accuracy": 0.92420988033139, + "macro avg": { + "precision": 0.9154441853604571, + "recall": 0.8316275984520265, + "f1-score": 0.8653422470187262, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.923033429798448, + "recall": 0.92420988033139, + "f1-score": 0.9199753884233416, + "support": 16295.0 + }, + "cross-entropy": 0.1776396237040874, + "roc-auc": 0.9632457301074769, + "score": 0.9632457301074769 + } + }, + "time": "0:07:13.032423", + "diversity": { + "train": { + "mean_centered_corr": 0.05698179081082344, + "mean_pairwise_disagreement": 0.03173294702843864, + "member_std": 0.031025612726807594 + }, + "val": { + "mean_centered_corr": 0.06959165632724762, + "mean_pairwise_disagreement": 0.030371885602247718, + "member_std": 0.033528175204992294 + }, + "test": { + "mean_centered_corr": 0.08432681858539581, + "mean_pairwise_disagreement": 0.03709145393896802, + "member_std": 0.03507966548204422 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 1, + 6, + 15, + 14, + 23, + 5, + 4, + 28, + 10, + 25, + 11, + 6, + 0, + 13, + 15, + 30, + 4, + 23, + 6 + ], + "head_selection_time": "0:00:26.623499" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-best-head-evaluation/0/report.json new file mode 100644 index 000000000..546302ba0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-best-head-evaluation/0/report.json @@ -0,0 +1,190 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22658863410177663, + "mae": 0.12739618122577667, + "r2": 0.6719876527786255, + "score": -0.22658863410177663 + }, + "val": { + "rmse": 0.23955454428788905, + "mae": 0.13915589451789856, + "r2": 0.6166812181472778, + "score": -0.23955454428788905 + }, + "test": { + "rmse": 0.26588264760298186, + "mae": 0.1863766759634018, + "r2": 0.5169194340705872, + "score": -0.26588264760298186 + } + }, + "time": "0:00:48.800887", + "diversity": { + "train": { + "mean_centered_corr": -0.018623020499944687, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05047355964779854 + }, + "val": { + "mean_centered_corr": -0.01870153285562992, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06864207983016968 + }, + "test": { + "mean_centered_corr": -0.019026711583137512, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09529124945402145 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015244" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-best-head-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-best-head-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-best-head-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-best-head-evaluation/1/report.json new file mode 100644 index 000000000..ace612a7a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-best-head-evaluation/1/report.json @@ -0,0 +1,190 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 962, + "metrics": { + "train": { + "rmse": 0.2225609541964927, + "mae": 0.13133738934993744, + "r2": 0.6835451126098633, + "score": -0.2225609541964927 + }, + "val": { + "rmse": 0.24150787640249508, + "mae": 0.13435126841068268, + "r2": 0.6104046106338501, + "score": -0.24150787640249508 + }, + "test": { + "rmse": 0.2416341058127328, + "mae": 0.15234874188899994, + "r2": 0.6010154485702515, + "score": -0.2416341058127328 + } + }, + "time": "0:00:53.667022", + "diversity": { + "train": { + "mean_centered_corr": -0.017433883622288704, + "mean_pairwise_disagreement": NaN, + "member_std": 0.049763333052396774 + }, + "val": { + "mean_centered_corr": -0.01871240884065628, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08149437606334686 + }, + "test": { + "mean_centered_corr": -0.018136458471417427, + "mean_pairwise_disagreement": NaN, + "member_std": 0.11388187855482101 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 28 + ], + "head_selection_time": "0:00:00.015286" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-best-head-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-best-head-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-best-head-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-best-head-evaluation/2/report.json new file mode 100644 index 000000000..a4d197bb5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-best-head-evaluation/2/report.json @@ -0,0 +1,190 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23639802450731898, + "mae": 0.12933363020420074, + "r2": 0.6429725885391235, + "score": -0.23639802450731898 + }, + "val": { + "rmse": 0.23944956044068919, + "mae": 0.13556918501853943, + "r2": 0.6170171499252319, + "score": -0.23944956044068919 + }, + "test": { + "rmse": 0.24308412455882378, + "mae": 0.15950754284858704, + "r2": 0.5962125658988953, + "score": -0.24308412455882378 + } + }, + "time": "0:00:45.117774", + "diversity": { + "train": { + "mean_centered_corr": -0.019429348409175873, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0477914959192276 + }, + "val": { + "mean_centered_corr": -0.017664093524217606, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06744762510061264 + }, + "test": { + "mean_centered_corr": -0.018886378034949303, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07553428411483765 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 30 + ], + "head_selection_time": "0:00:00.015062" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-evaluation/0.toml new file mode 100644 index 000000000..a68e82651 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-evaluation/0.toml @@ -0,0 +1,34 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-evaluation/0/report.json new file mode 100644 index 000000000..c4a6eed6f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-evaluation/0/report.json @@ -0,0 +1,186 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22224988999004666, + "mae": 0.11853151023387909, + "r2": 0.6844290494918823, + "score": -0.22224988999004666 + }, + "val": { + "rmse": 0.23452539387100468, + "mae": 0.12745676934719086, + "r2": 0.6326068639755249, + "score": -0.23452539387100468 + }, + "test": { + "rmse": 0.23846659721529556, + "mae": 0.15488465130329132, + "r2": 0.6114072799682617, + "score": -0.23846659721529556 + } + }, + "time": "0:00:48.800887", + "diversity": { + "train": { + "mean_centered_corr": -0.018623020499944687, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05047355964779854 + }, + "val": { + "mean_centered_corr": -0.01870153285562992, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06864207983016968 + }, + "test": { + "mean_centered_corr": -0.019026711583137512, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09529124945402145 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-evaluation/1.toml new file mode 100644 index 000000000..b6864e8d9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-evaluation/1.toml @@ -0,0 +1,34 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 1 +head_selection = true + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-evaluation/1/report.json new file mode 100644 index 000000000..59ef4bb78 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-evaluation/1/report.json @@ -0,0 +1,186 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 962, + "metrics": { + "train": { + "rmse": 0.21503383723674574, + "mae": 0.11624167859554291, + "r2": 0.7045884132385254, + "score": -0.21503383723674574 + }, + "val": { + "rmse": 0.23623006536000674, + "mae": 0.1255212128162384, + "r2": 0.627246618270874, + "score": -0.23623006536000674 + }, + "test": { + "rmse": 0.2318497676950212, + "mae": 0.13931861519813538, + "r2": 0.632672905921936, + "score": -0.2318497676950212 + } + }, + "time": "0:00:53.667022", + "diversity": { + "train": { + "mean_centered_corr": -0.017433883622288704, + "mean_pairwise_disagreement": NaN, + "member_std": 0.049763333052396774 + }, + "val": { + "mean_centered_corr": -0.01871240884065628, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08149437606334686 + }, + "test": { + "mean_centered_corr": -0.018136458471417427, + "mean_pairwise_disagreement": NaN, + "member_std": 0.11388187855482101 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-evaluation/2.toml new file mode 100644 index 000000000..e02a76aa5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-evaluation/2.toml @@ -0,0 +1,34 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 2 +head_selection = true + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-evaluation/2/report.json new file mode 100644 index 000000000..97b454f47 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-evaluation/2/report.json @@ -0,0 +1,186 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23231268970798877, + "mae": 0.12570486962795258, + "r2": 0.6552059650421143, + "score": -0.23231268970798877 + }, + "val": { + "rmse": 0.23581900020763122, + "mae": 0.12935329973697662, + "r2": 0.6285427808761597, + "score": -0.23581900020763122 + }, + "test": { + "rmse": 0.23293721743587778, + "mae": 0.1457630842924118, + "r2": 0.6292190551757812, + "score": -0.23293721743587778 + } + }, + "time": "0:00:45.117774", + "diversity": { + "train": { + "mean_centered_corr": -0.019429348409175873, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0477914959192276 + }, + "val": { + "mean_centered_corr": -0.017664093524217606, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06744762510061264 + }, + "test": { + "mean_centered_corr": -0.018886378034949303, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07553428411483765 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..d2edb64f2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/0/report.json @@ -0,0 +1,199 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22302489965783162, + "mae": 0.11715596914291382, + "r2": 0.6822243928909302, + "score": -0.22302489965783162 + }, + "val": { + "rmse": 0.23257941028161666, + "mae": 0.12573255598545074, + "r2": 0.6386785507202148, + "score": -0.23257941028161666 + }, + "test": { + "rmse": 0.23549883338311248, + "mae": 0.14942243695259094, + "r2": 0.6210193037986755, + "score": -0.23549883338311248 + } + }, + "time": "0:00:48.800887", + "diversity": { + "train": { + "mean_centered_corr": -0.018623020499944687, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05047355964779854 + }, + "val": { + "mean_centered_corr": -0.01870153285562992, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06864207983016968 + }, + "test": { + "mean_centered_corr": -0.019026711583137512, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09529124945402145 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 19, + 21, + 12, + 16, + 18, + 8, + 22, + 19, + 29 + ], + "head_selection_time": "0:00:00.158388" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/1/report.json new file mode 100644 index 000000000..bbce78c6f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/1/report.json @@ -0,0 +1,200 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 962, + "metrics": { + "train": { + "rmse": 0.21477064276002938, + "mae": 0.1157485619187355, + "r2": 0.7053111791610718, + "score": -0.21477064276002938 + }, + "val": { + "rmse": 0.2341956008327552, + "mae": 0.12569019198417664, + "r2": 0.6336394548416138, + "score": -0.2341956008327552 + }, + "test": { + "rmse": 0.23089961816224172, + "mae": 0.13695372641086578, + "r2": 0.6356774568557739, + "score": -0.23089961816224172 + } + }, + "time": "0:00:53.667022", + "diversity": { + "train": { + "mean_centered_corr": -0.017433883622288704, + "mean_pairwise_disagreement": NaN, + "member_std": 0.049763333052396774 + }, + "val": { + "mean_centered_corr": -0.01871240884065628, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08149437606334686 + }, + "test": { + "mean_centered_corr": -0.018136458471417427, + "mean_pairwise_disagreement": NaN, + "member_std": 0.11388187855482101 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 28, + 6, + 21, + 1, + 31, + 13, + 14, + 21, + 6, + 15, + 5 + ], + "head_selection_time": "0:00:00.175132" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/2/report.json new file mode 100644 index 000000000..e34fa6bb4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/2/report.json @@ -0,0 +1,201 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23317289428038254, + "mae": 0.12557387351989746, + "r2": 0.652647852897644, + "score": -0.23317289428038254 + }, + "val": { + "rmse": 0.23449523543792233, + "mae": 0.12881776690483093, + "r2": 0.6327013969421387, + "score": -0.23449523543792233 + }, + "test": { + "rmse": 0.23591672540617004, + "mae": 0.1523103564977646, + "r2": 0.6196731328964233, + "score": -0.23591672540617004 + } + }, + "time": "0:00:45.117774", + "diversity": { + "train": { + "mean_centered_corr": -0.019429348409175873, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0477914959192276 + }, + "val": { + "mean_centered_corr": -0.017664093524217606, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06744762510061264 + }, + "test": { + "mean_centered_corr": -0.018886378034949303, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07553428411483765 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 30, + 27, + 9, + 4, + 31, + 5, + 12, + 2, + 22, + 13, + 2, + 4 + ], + "head_selection_time": "0:00:00.188219" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..bfcad143f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-best-head-evaluation/0/report.json @@ -0,0 +1,643 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24287177140617242, + "mae": 0.12792739272117615, + "r2": 0.6231504678726196, + "score": -0.24287177140617242 + }, + "val": { + "rmse": 0.2388197411165918, + "mae": 0.13359415531158447, + "r2": 0.61902916431427, + "score": -0.2388197411165918 + }, + "test": { + "rmse": 0.2462523116085554, + "mae": 0.16663780808448792, + "r2": 0.5856186151504517, + "score": -0.2462523116085554 + } + }, + "time": "0:01:03.015496", + "diversity": { + "train": { + "mean_centered_corr": -0.018840299919247627, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04810040444135666 + }, + "val": { + "mean_centered_corr": -0.020352300256490707, + "mean_pairwise_disagreement": NaN, + "member_std": 0.061659909784793854 + }, + "test": { + "mean_centered_corr": -0.019725581631064415, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08170976489782333 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015354" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-best-head-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-best-head-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-best-head-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-best-head-evaluation/1/report.json new file mode 100644 index 000000000..868c3077d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-best-head-evaluation/1/report.json @@ -0,0 +1,643 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.23314741034598338, + "mae": 0.12273190170526505, + "r2": 0.6527237892150879, + "score": -0.23314741034598338 + }, + "val": { + "rmse": 0.24061445857155628, + "mae": 0.13311389088630676, + "r2": 0.6132817268371582, + "score": -0.24061445857155628 + }, + "test": { + "rmse": 0.23970045288478414, + "mae": 0.14821729063987732, + "r2": 0.6073756217956543, + "score": -0.23970045288478414 + } + }, + "time": "0:01:09.119012", + "diversity": { + "train": { + "mean_centered_corr": -0.018567770719528198, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0464579202234745 + }, + "val": { + "mean_centered_corr": -0.01797967404127121, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06989888846874237 + }, + "test": { + "mean_centered_corr": -0.01808270625770092, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09372883290052414 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 27 + ], + "head_selection_time": "0:00:00.015071" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-best-head-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-best-head-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-best-head-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-best-head-evaluation/2/report.json new file mode 100644 index 000000000..d12d59a2d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-best-head-evaluation/2/report.json @@ -0,0 +1,643 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 518, + "metrics": { + "train": { + "rmse": 0.24266637864500767, + "mae": 0.1298421323299408, + "r2": 0.6237875819206238, + "score": -0.24266637864500767 + }, + "val": { + "rmse": 0.2376592052308581, + "mae": 0.13035762310028076, + "r2": 0.6227228045463562, + "score": -0.2376592052308581 + }, + "test": { + "rmse": 0.23311093711112285, + "mae": 0.13708607852458954, + "r2": 0.6286658644676208, + "score": -0.23311093711112285 + } + }, + "time": "0:00:57.753537", + "diversity": { + "train": { + "mean_centered_corr": -0.019510088488459587, + "mean_pairwise_disagreement": NaN, + "member_std": 0.044065676629543304 + }, + "val": { + "mean_centered_corr": -0.020489495247602463, + "mean_pairwise_disagreement": NaN, + "member_std": 0.058154650032520294 + }, + "test": { + "mean_centered_corr": -0.02033652924001217, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06957374513149261 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 27 + ], + "head_selection_time": "0:00:00.015257" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/0.toml new file mode 100644 index 000000000..523756861 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/0.toml @@ -0,0 +1,45 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/0/report.json new file mode 100644 index 000000000..747fa815c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/0/report.json @@ -0,0 +1,639 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23279361488245573, + "mae": 0.12178773432970047, + "r2": 0.6537768840789795, + "score": -0.23279361488245573 + }, + "val": { + "rmse": 0.23523413233394516, + "mae": 0.12663741409778595, + "r2": 0.6303830146789551, + "score": -0.23523413233394516 + }, + "test": { + "rmse": 0.235149605552611, + "mae": 0.14915932714939117, + "r2": 0.6221424341201782, + "score": -0.235149605552611 + } + }, + "time": "0:01:03.015496", + "diversity": { + "train": { + "mean_centered_corr": -0.018840299919247627, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04810040444135666 + }, + "val": { + "mean_centered_corr": -0.020352300256490707, + "mean_pairwise_disagreement": NaN, + "member_std": 0.061659909784793854 + }, + "test": { + "mean_centered_corr": -0.019725581631064415, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08170976489782333 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/1.toml new file mode 100644 index 000000000..66c960456 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/1.toml @@ -0,0 +1,45 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 1 +head_selection = true + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/1/report.json new file mode 100644 index 000000000..5e3b9688f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/1/report.json @@ -0,0 +1,639 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22716695623112323, + "mae": 0.11858279258012772, + "r2": 0.6703112125396729, + "score": -0.22716695623112323 + }, + "val": { + "rmse": 0.23639255622691643, + "mae": 0.12672214210033417, + "r2": 0.626733660697937, + "score": -0.23639255622691643 + }, + "test": { + "rmse": 0.2324256095265638, + "mae": 0.1419195681810379, + "r2": 0.6308460235595703, + "score": -0.2324256095265638 + } + }, + "time": "0:01:09.119012", + "diversity": { + "train": { + "mean_centered_corr": -0.018567770719528198, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0464579202234745 + }, + "val": { + "mean_centered_corr": -0.01797967404127121, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06989888846874237 + }, + "test": { + "mean_centered_corr": -0.01808270625770092, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09372883290052414 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/2.toml new file mode 100644 index 000000000..69535992d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/2.toml @@ -0,0 +1,45 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 2 +head_selection = true + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/2/report.json new file mode 100644 index 000000000..f2a4c0d43 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/2/report.json @@ -0,0 +1,639 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 518, + "metrics": { + "train": { + "rmse": 0.24112485206623196, + "mae": 0.12558841705322266, + "r2": 0.6285521388053894, + "score": -0.24112485206623196 + }, + "val": { + "rmse": 0.2359054032023901, + "mae": 0.12690992653369904, + "r2": 0.6282705068588257, + "score": -0.2359054032023901 + }, + "test": { + "rmse": 0.23245970634502372, + "mae": 0.14181140065193176, + "r2": 0.6307376623153687, + "score": -0.23245970634502372 + } + }, + "time": "0:00:57.753537", + "diversity": { + "train": { + "mean_centered_corr": -0.019510088488459587, + "mean_pairwise_disagreement": NaN, + "member_std": 0.044065676629543304 + }, + "val": { + "mean_centered_corr": -0.020489495247602463, + "mean_pairwise_disagreement": NaN, + "member_std": 0.058154650032520294 + }, + "test": { + "mean_centered_corr": -0.02033652924001217, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06957374513149261 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..d85be1baa --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,649 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23477394006764257, + "mae": 0.1248951405286789, + "r2": 0.6478613615036011, + "score": -0.23477394006764257 + }, + "val": { + "rmse": 0.23327114512588593, + "mae": 0.12512965500354767, + "r2": 0.6365260481834412, + "score": -0.23327114512588593 + }, + "test": { + "rmse": 0.23367141178994547, + "mae": 0.14663751423358917, + "r2": 0.6268780827522278, + "score": -0.23367141178994547 + } + }, + "time": "0:01:03.015496", + "diversity": { + "train": { + "mean_centered_corr": -0.018840299919247627, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04810040444135666 + }, + "val": { + "mean_centered_corr": -0.020352300256490707, + "mean_pairwise_disagreement": NaN, + "member_std": 0.061659909784793854 + }, + "test": { + "mean_centered_corr": -0.019725581631064415, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08170976489782333 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 0, + 21, + 25, + 19, + 0, + 28 + ], + "head_selection_time": "0:00:00.116743" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/1/report.json new file mode 100644 index 000000000..cba3c4596 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/1/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2278723424771184, + "mae": 0.11909302324056625, + "r2": 0.6682605743408203, + "score": -0.2278723424771184 + }, + "val": { + "rmse": 0.23463880474397839, + "mae": 0.12749379873275757, + "r2": 0.632251501083374, + "score": -0.23463880474397839 + }, + "test": { + "rmse": 0.2331063026442073, + "mae": 0.1443300098180771, + "r2": 0.6286805868148804, + "score": -0.2331063026442073 + } + }, + "time": "0:01:09.119012", + "diversity": { + "train": { + "mean_centered_corr": -0.018567770719528198, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0464579202234745 + }, + "val": { + "mean_centered_corr": -0.01797967404127121, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06989888846874237 + }, + "test": { + "mean_centered_corr": -0.01808270625770092, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09372883290052414 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 27, + 1, + 19, + 31, + 14, + 1, + 6, + 29, + 16, + 31, + 4, + 1, + 1, + 3, + 29, + 19 + ], + "head_selection_time": "0:00:00.250837" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/2/report.json new file mode 100644 index 000000000..e8f693d35 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/2/report.json @@ -0,0 +1,648 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 518, + "metrics": { + "train": { + "rmse": 0.24263932781325961, + "mae": 0.1284457892179489, + "r2": 0.6238714456558228, + "score": -0.24263932781325961 + }, + "val": { + "rmse": 0.23432768503071347, + "mae": 0.12745344638824463, + "r2": 0.6332260370254517, + "score": -0.23432768503071347 + }, + "test": { + "rmse": 0.2325014731401514, + "mae": 0.14458957314491272, + "r2": 0.6306049823760986, + "score": -0.2325014731401514 + } + }, + "time": "0:00:57.753537", + "diversity": { + "train": { + "mean_centered_corr": -0.019510088488459587, + "mean_pairwise_disagreement": NaN, + "member_std": 0.044065676629543304 + }, + "val": { + "mean_centered_corr": -0.020489495247602463, + "mean_pairwise_disagreement": NaN, + "member_std": 0.058154650032520294 + }, + "test": { + "mean_centered_corr": -0.02033652924001217, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06957374513149261 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 27, + 9, + 22, + 5, + 12, + 30 + ], + "head_selection_time": "0:00:00.100965" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/0/report.json new file mode 100644 index 000000000..022c71440 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/0/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24039030054595545, + "mae": 0.1237230896949768, + "r2": 0.6308118104934692, + "score": -0.24039030054595545 + }, + "val": { + "rmse": 0.23560357747729638, + "mae": 0.13029368221759796, + "r2": 0.6292210817337036, + "score": -0.23560357747729638 + }, + "test": { + "rmse": 0.2389191702722735, + "mae": 0.15265432000160217, + "r2": 0.6099308729171753, + "score": -0.2389191702722735 + } + }, + "time": "0:01:13.814362", + "diversity": { + "train": { + "mean_centered_corr": -0.018871087580919266, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04625045508146286 + }, + "val": { + "mean_centered_corr": -0.019568709656596184, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0614706352353096 + }, + "test": { + "mean_centered_corr": -0.018940167501568794, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08073103427886963 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015454" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/1/report.json new file mode 100644 index 000000000..051fe52d7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/1/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + }, + "mask_stats": { + "keep_rate_actual": 0.9749681353569031, + "min_features_per_member": 376.0, + "mean_features_per_member": 382.1875, + "max_features_per_member": 388.0, + "min_members_per_feature": 28.0, + "mean_members_per_feature": 31.1989803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9511310051056044, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1184, + "metrics": { + "train": { + "rmse": 0.2171563784853777, + "mae": 0.11889506876468658, + "r2": 0.6987277865409851, + "score": -0.2171563784853777 + }, + "val": { + "rmse": 0.24254141605050564, + "mae": 0.13972081243991852, + "r2": 0.607062816619873, + "score": -0.24254141605050564 + }, + "test": { + "rmse": 0.24204913155172003, + "mae": 0.15291987359523773, + "r2": 0.5996437072753906, + "score": -0.24204913155172003 + } + }, + "time": "0:01:33.744311", + "diversity": { + "train": { + "mean_centered_corr": -0.017405925318598747, + "mean_pairwise_disagreement": NaN, + "member_std": 0.049276966601610184 + }, + "val": { + "mean_centered_corr": -0.017822835594415665, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07913164794445038 + }, + "test": { + "mean_centered_corr": -0.01794285699725151, + "mean_pairwise_disagreement": NaN, + "member_std": 0.10268954932689667 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 27 + ], + "head_selection_time": "0:00:00.015010" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/2/report.json new file mode 100644 index 000000000..bc47fc492 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/2/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + }, + "mask_stats": { + "keep_rate_actual": 0.9760841727256775, + "min_features_per_member": 372.0, + "mean_features_per_member": 382.625, + "max_features_per_member": 390.0, + "min_members_per_feature": 28.0, + "mean_members_per_feature": 31.23469352722168, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9532826281104112, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2229962847735884, + "mae": 0.12252119928598404, + "r2": 0.6823059320449829, + "score": -0.2229962847735884 + }, + "val": { + "rmse": 0.23847619665119374, + "mae": 0.134144127368927, + "r2": 0.6201244592666626, + "score": -0.23847619665119374 + }, + "test": { + "rmse": 0.2542706881867437, + "mae": 0.17485685646533966, + "r2": 0.5581934452056885, + "score": -0.2542706881867437 + } + }, + "time": "0:01:19.517367", + "diversity": { + "train": { + "mean_centered_corr": -0.018493393436074257, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04774053022265434 + }, + "val": { + "mean_centered_corr": -0.018883541226387024, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07628075778484344 + }, + "test": { + "mean_centered_corr": -0.01829403080046177, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09460576623678207 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 24 + ], + "head_selection_time": "0:00:00.015325" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/0.toml new file mode 100644 index 000000000..aff429e28 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/0/report.json new file mode 100644 index 000000000..193998f2e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/0/report.json @@ -0,0 +1,684 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23278188472143527, + "mae": 0.12037675827741623, + "r2": 0.6538118124008179, + "score": -0.23278188472143527 + }, + "val": { + "rmse": 0.235050071622046, + "mae": 0.12535566091537476, + "r2": 0.6309611797332764, + "score": -0.235050071622046 + }, + "test": { + "rmse": 0.23184680318718037, + "mae": 0.14313000440597534, + "r2": 0.6326823234558105, + "score": -0.23184680318718037 + } + }, + "time": "0:01:13.814362", + "diversity": { + "train": { + "mean_centered_corr": -0.018871087580919266, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04625045508146286 + }, + "val": { + "mean_centered_corr": -0.019568709656596184, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0614706352353096 + }, + "test": { + "mean_centered_corr": -0.018940167501568794, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08073103427886963 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/1.toml new file mode 100644 index 000000000..ad01c71b3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/1.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 1 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 1 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/1/report.json new file mode 100644 index 000000000..f4c5d18c3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/1/report.json @@ -0,0 +1,684 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + }, + "mask_stats": { + "keep_rate_actual": 0.9749681353569031, + "min_features_per_member": 376.0, + "mean_features_per_member": 382.1875, + "max_features_per_member": 388.0, + "min_members_per_feature": 28.0, + "mean_members_per_feature": 31.1989803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9511310051056044, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1184, + "metrics": { + "train": { + "rmse": 0.20665772048240127, + "mae": 0.11000452935695648, + "r2": 0.7271542549133301, + "score": -0.20665772048240127 + }, + "val": { + "rmse": 0.23618618194018487, + "mae": 0.12565438449382782, + "r2": 0.6273850798606873, + "score": -0.23618618194018487 + }, + "test": { + "rmse": 0.23253236276211406, + "mae": 0.1399872750043869, + "r2": 0.6305068731307983, + "score": -0.23253236276211406 + } + }, + "time": "0:01:33.744311", + "diversity": { + "train": { + "mean_centered_corr": -0.017405925318598747, + "mean_pairwise_disagreement": NaN, + "member_std": 0.049276966601610184 + }, + "val": { + "mean_centered_corr": -0.017822835594415665, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07913164794445038 + }, + "test": { + "mean_centered_corr": -0.01794285699725151, + "mean_pairwise_disagreement": NaN, + "member_std": 0.10268954932689667 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/2.toml new file mode 100644 index 000000000..86addbb91 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/2.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 2 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 2 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/2/report.json new file mode 100644 index 000000000..65d6d6497 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/2/report.json @@ -0,0 +1,684 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + }, + "mask_stats": { + "keep_rate_actual": 0.9760841727256775, + "min_features_per_member": 372.0, + "mean_features_per_member": 382.625, + "max_features_per_member": 390.0, + "min_members_per_feature": 28.0, + "mean_members_per_feature": 31.23469352722168, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9532826281104112, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.21767157152228817, + "mae": 0.11745961010456085, + "r2": 0.6972966194152832, + "score": -0.21767157152228817 + }, + "val": { + "rmse": 0.23592642069691294, + "mae": 0.12567678093910217, + "r2": 0.6282042264938354, + "score": -0.23592642069691294 + }, + "test": { + "rmse": 0.2326191697888986, + "mae": 0.14303357899188995, + "r2": 0.6302309036254883, + "score": -0.2326191697888986 + } + }, + "time": "0:01:19.517367", + "diversity": { + "train": { + "mean_centered_corr": -0.018493393436074257, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04774053022265434 + }, + "val": { + "mean_centered_corr": -0.018883541226387024, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07628075778484344 + }, + "test": { + "mean_centered_corr": -0.01829403080046177, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09460576623678207 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..55f69c783 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/0/report.json @@ -0,0 +1,692 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23573796221874224, + "mae": 0.12117554992437363, + "r2": 0.6449635028839111, + "score": -0.23573796221874224 + }, + "val": { + "rmse": 0.2324412121462238, + "mae": 0.12522703409194946, + "r2": 0.6391078233718872, + "score": -0.2324412121462238 + }, + "test": { + "rmse": 0.23820740696857753, + "mae": 0.1556985229253769, + "r2": 0.6122515201568604, + "score": -0.23820740696857753 + } + }, + "time": "0:01:13.814362", + "diversity": { + "train": { + "mean_centered_corr": -0.018871087580919266, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04625045508146286 + }, + "val": { + "mean_centered_corr": -0.019568709656596184, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0614706352353096 + }, + "test": { + "mean_centered_corr": -0.018940167501568794, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08073103427886963 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 0, + 1, + 25, + 21 + ], + "head_selection_time": "0:00:00.088240" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/1/report.json new file mode 100644 index 000000000..9dcd68238 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/1/report.json @@ -0,0 +1,703 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + }, + "mask_stats": { + "keep_rate_actual": 0.9749681353569031, + "min_features_per_member": 376.0, + "mean_features_per_member": 382.1875, + "max_features_per_member": 388.0, + "min_members_per_feature": 28.0, + "mean_members_per_feature": 31.1989803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9511310051056044, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1184, + "metrics": { + "train": { + "rmse": 0.20580531574085753, + "mae": 0.11071916669607162, + "r2": 0.7294004559516907, + "score": -0.20580531574085753 + }, + "val": { + "rmse": 0.23397077829789442, + "mae": 0.12587550282478333, + "r2": 0.6343424916267395, + "score": -0.23397077829789442 + }, + "test": { + "rmse": 0.23276530465893142, + "mae": 0.13940665125846863, + "r2": 0.6297661662101746, + "score": -0.23276530465893142 + } + }, + "time": "0:01:33.744311", + "diversity": { + "train": { + "mean_centered_corr": -0.017405925318598747, + "mean_pairwise_disagreement": NaN, + "member_std": 0.049276966601610184 + }, + "val": { + "mean_centered_corr": -0.017822835594415665, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07913164794445038 + }, + "test": { + "mean_centered_corr": -0.01794285699725151, + "mean_pairwise_disagreement": NaN, + "member_std": 0.10268954932689667 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 27, + 6, + 1, + 15, + 2, + 13, + 0, + 6, + 1, + 29, + 31, + 5, + 1, + 21, + 6, + 22 + ], + "head_selection_time": "0:00:00.247440" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/2/report.json new file mode 100644 index 000000000..88ac1a9ce --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/2/report.json @@ -0,0 +1,696 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + }, + "mask_stats": { + "keep_rate_actual": 0.9760841727256775, + "min_features_per_member": 372.0, + "mean_features_per_member": 382.625, + "max_features_per_member": 390.0, + "min_members_per_feature": 28.0, + "mean_members_per_feature": 31.23469352722168, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9532826281104112, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2210309033660134, + "mae": 0.12036030739545822, + "r2": 0.6878812313079834, + "score": -0.2210309033660134 + }, + "val": { + "rmse": 0.23411027740292373, + "mae": 0.128285214304924, + "r2": 0.633906364440918, + "score": -0.23411027740292373 + }, + "test": { + "rmse": 0.24496736545125247, + "mae": 0.16705775260925293, + "r2": 0.589931845664978, + "score": -0.24496736545125247 + } + }, + "time": "0:01:19.517367", + "diversity": { + "train": { + "mean_centered_corr": -0.018493393436074257, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04774053022265434 + }, + "val": { + "mean_centered_corr": -0.018883541226387024, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07628075778484344 + }, + "test": { + "mean_centered_corr": -0.01829403080046177, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09460576623678207 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 24, + 4, + 27, + 6, + 3, + 30, + 5, + 27, + 4 + ], + "head_selection_time": "0:00:00.145674" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..bc82de281 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-best-head-evaluation/0/report.json @@ -0,0 +1,200 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-evaluation/_esam_diagnostics/esam_rho0.0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2261742520907439, + "mae": 0.12224072217941284, + "r2": 0.6731863021850586, + "score": -0.2261742520907439 + }, + "val": { + "rmse": 0.2377136850270048, + "mae": 0.13327811658382416, + "r2": 0.622549831867218, + "score": -0.2377136850270048 + }, + "test": { + "rmse": 0.262345673404661, + "mae": 0.17990553379058838, + "r2": 0.5296865105628967, + "score": -0.262345673404661 + } + }, + "time": "0:01:27.301753", + "diversity": { + "train": { + "mean_centered_corr": -0.01859835349023342, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04920431599020958 + }, + "val": { + "mean_centered_corr": -0.019710257649421692, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06811030209064484 + }, + "test": { + "mean_centered_corr": -0.013359437696635723, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09920918941497803 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015122" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-best-head-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-best-head-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-best-head-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-best-head-evaluation/1/report.json new file mode 100644 index 000000000..8e1a4d015 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-best-head-evaluation/1/report.json @@ -0,0 +1,200 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__1.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2294608480279213, + "mae": 0.12013265490531921, + "r2": 0.6636192798614502, + "score": -0.2294608480279213 + }, + "val": { + "rmse": 0.2385787982165438, + "mae": 0.13280385732650757, + "r2": 0.6197975277900696, + "score": -0.2385787982165438 + }, + "test": { + "rmse": 0.24822352719647534, + "mae": 0.16104964911937714, + "r2": 0.5789579153060913, + "score": -0.24822352719647534 + } + }, + "time": "0:01:28.517490", + "diversity": { + "train": { + "mean_centered_corr": -0.01827217824757099, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04794318601489067 + }, + "val": { + "mean_centered_corr": -0.019240442663431168, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07059209793806076 + }, + "test": { + "mean_centered_corr": -0.020052773877978325, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09476596862077713 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 1 + ], + "head_selection_time": "0:00:00.015630" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-best-head-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-best-head-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-best-head-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-best-head-evaluation/2/report.json new file mode 100644 index 000000000..e689e8acb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-best-head-evaluation/2/report.json @@ -0,0 +1,200 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__2.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22841502689654178, + "mae": 0.12657804787158966, + "r2": 0.6666785478591919, + "score": -0.22841502689654178 + }, + "val": { + "rmse": 0.24089231567126032, + "mae": 0.1354312300682068, + "r2": 0.612388014793396, + "score": -0.24089231567126032 + }, + "test": { + "rmse": 0.24860647860679874, + "mae": 0.16609469056129456, + "r2": 0.5776578187942505, + "score": -0.24860647860679874 + } + }, + "time": "0:01:27.073529", + "diversity": { + "train": { + "mean_centered_corr": -0.019437065348029137, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05152730643749237 + }, + "val": { + "mean_centered_corr": -0.018969101831316948, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07230502367019653 + }, + "test": { + "mean_centered_corr": -0.018757972866296768, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08676532655954361 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 30 + ], + "head_selection_time": "0:00:00.015302" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-evaluation/0.toml new file mode 100644 index 000000000..05c49e679 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-evaluation/0.toml @@ -0,0 +1,44 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.0025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-evaluation/0/report.json new file mode 100644 index 000000000..5fee35a30 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-evaluation/0/report.json @@ -0,0 +1,196 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-evaluation/_esam_diagnostics/esam_rho0.0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22265827445732783, + "mae": 0.11788544058799744, + "r2": 0.6832683086395264, + "score": -0.22265827445732783 + }, + "val": { + "rmse": 0.23474623366768377, + "mae": 0.1270996779203415, + "r2": 0.6319146752357483, + "score": -0.23474623366768377 + }, + "test": { + "rmse": 0.23936563514587833, + "mae": 0.15535929799079895, + "r2": 0.6084716320037842, + "score": -0.23936563514587833 + } + }, + "time": "0:01:27.301753", + "diversity": { + "train": { + "mean_centered_corr": -0.01859835349023342, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04920431599020958 + }, + "val": { + "mean_centered_corr": -0.019710257649421692, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06811030209064484 + }, + "test": { + "mean_centered_corr": -0.013359437696635723, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09920918941497803 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-evaluation/1.toml new file mode 100644 index 000000000..14067822a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-evaluation/1.toml @@ -0,0 +1,44 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 1 +head_selection = true +use_esam = true +esam_rho = 0.0025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-evaluation/1/report.json new file mode 100644 index 000000000..d8fdf451a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-evaluation/1/report.json @@ -0,0 +1,196 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__1.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22373909069883033, + "mae": 0.11784303933382034, + "r2": 0.6801859140396118, + "score": -0.22373909069883033 + }, + "val": { + "rmse": 0.23723213436744667, + "mae": 0.12606672942638397, + "r2": 0.624077558517456, + "score": -0.23723213436744667 + }, + "test": { + "rmse": 0.23099161094856513, + "mae": 0.13413691520690918, + "r2": 0.635387122631073, + "score": -0.23099161094856513 + } + }, + "time": "0:01:28.517490", + "diversity": { + "train": { + "mean_centered_corr": -0.01827217824757099, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04794318601489067 + }, + "val": { + "mean_centered_corr": -0.019240442663431168, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07059209793806076 + }, + "test": { + "mean_centered_corr": -0.020052773877978325, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09476596862077713 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-evaluation/2.toml new file mode 100644 index 000000000..0aaff1748 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-evaluation/2.toml @@ -0,0 +1,44 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 2 +head_selection = true +use_esam = true +esam_rho = 0.0025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-evaluation/2/report.json new file mode 100644 index 000000000..d8eac53d6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-evaluation/2/report.json @@ -0,0 +1,196 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__2.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22303537248925367, + "mae": 0.12042270600795746, + "r2": 0.6821945309638977, + "score": -0.22303537248925367 + }, + "val": { + "rmse": 0.2356560348188032, + "mae": 0.12620244920253754, + "r2": 0.6290559768676758, + "score": -0.2356560348188032 + }, + "test": { + "rmse": 0.23118945759761478, + "mae": 0.1404353380203247, + "r2": 0.6347622871398926, + "score": -0.23118945759761478 + } + }, + "time": "0:01:27.073529", + "diversity": { + "train": { + "mean_centered_corr": -0.019437065348029137, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05152730643749237 + }, + "val": { + "mean_centered_corr": -0.018969101831316948, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07230502367019653 + }, + "test": { + "mean_centered_corr": -0.018757972866296768, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08676532655954361 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..9ca33d310 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,207 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-evaluation/_esam_diagnostics/esam_rho0.0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22450348389711117, + "mae": 0.11922673135995865, + "r2": 0.6779968738555908, + "score": -0.22450348389711117 + }, + "val": { + "rmse": 0.23295802298295287, + "mae": 0.1258699893951416, + "r2": 0.6375011801719666, + "score": -0.23295802298295287 + }, + "test": { + "rmse": 0.23903366020873654, + "mae": 0.15511320531368256, + "r2": 0.6095569133758545, + "score": -0.23903366020873654 + } + }, + "time": "0:01:27.301753", + "diversity": { + "train": { + "mean_centered_corr": -0.01859835349023342, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04920431599020958 + }, + "val": { + "mean_centered_corr": -0.019710257649421692, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06811030209064484 + }, + "test": { + "mean_centered_corr": -0.013359437696635723, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09920918941497803 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 22, + 12, + 16, + 21, + 19, + 25, + 29 + ], + "head_selection_time": "0:00:00.128997" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/1/report.json new file mode 100644 index 000000000..881467a21 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/1/report.json @@ -0,0 +1,206 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__1.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2241471801070519, + "mae": 0.11760734766721725, + "r2": 0.6790181398391724, + "score": -0.2241471801070519 + }, + "val": { + "rmse": 0.23392588182013857, + "mae": 0.12585178017616272, + "r2": 0.6344828009605408, + "score": -0.23392588182013857 + }, + "test": { + "rmse": 0.23227402448302542, + "mae": 0.14325864613056183, + "r2": 0.6313273906707764, + "score": -0.23227402448302542 + } + }, + "time": "0:01:28.517490", + "diversity": { + "train": { + "mean_centered_corr": -0.01827217824757099, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04794318601489067 + }, + "val": { + "mean_centered_corr": -0.019240442663431168, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07059209793806076 + }, + "test": { + "mean_centered_corr": -0.020052773877978325, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09476596862077713 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 1, + 21, + 28, + 22, + 0, + 6, + 13 + ], + "head_selection_time": "0:00:00.124247" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/2/report.json new file mode 100644 index 000000000..96f1d4f48 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/2/report.json @@ -0,0 +1,205 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__2.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2242032567429998, + "mae": 0.12045589089393616, + "r2": 0.6788575649261475, + "score": -0.2242032567429998 + }, + "val": { + "rmse": 0.23456056719125185, + "mae": 0.12557265162467957, + "r2": 0.63249671459198, + "score": -0.23456056719125185 + }, + "test": { + "rmse": 0.23191054380220286, + "mae": 0.14388112723827362, + "r2": 0.6324803829193115, + "score": -0.23191054380220286 + } + }, + "time": "0:01:27.073529", + "diversity": { + "train": { + "mean_centered_corr": -0.019437065348029137, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05152730643749237 + }, + "val": { + "mean_centered_corr": -0.018969101831316948, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07230502367019653 + }, + "test": { + "mean_centered_corr": -0.018757972866296768, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08676532655954361 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 30, + 12, + 5, + 27, + 9, + 15 + ], + "head_selection_time": "0:00:00.106322" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..688b587ad --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-best-head-evaluation/0/report.json @@ -0,0 +1,226 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.8038105964660645, + "min_features_per_member": 298.0, + "mean_features_per_member": 315.09375, + "max_features_per_member": 335.0, + "min_members_per_feature": 19.0, + "mean_members_per_feature": 25.721939086914062, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.6715097166219348, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2316786318343145, + "mae": 0.12621846795082092, + "r2": 0.6570855379104614, + "score": -0.2316786318343145 + }, + "val": { + "rmse": 0.2380346366558266, + "mae": 0.1380215585231781, + "r2": 0.6215299367904663, + "score": -0.2380346366558266 + }, + "test": { + "rmse": 0.2579307429494123, + "mae": 0.17571206390857697, + "r2": 0.5453828573226929, + "score": -0.2579307429494123 + } + }, + "time": "0:00:59.854964", + "diversity": { + "train": { + "mean_centered_corr": -0.017583154141902924, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05003640055656433 + }, + "val": { + "mean_centered_corr": -0.018760371953248978, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07571490854024887 + }, + "test": { + "mean_centered_corr": -0.019624220207333565, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09894119203090668 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25 + ], + "head_selection_time": "0:00:00.016145" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-best-head-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-best-head-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-best-head-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-best-head-evaluation/1/report.json new file mode 100644 index 000000000..6f8453216 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-best-head-evaluation/1/report.json @@ -0,0 +1,226 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + }, + "mask_stats": { + "keep_rate_actual": 0.8027742505073547, + "min_features_per_member": 296.0, + "mean_features_per_member": 314.6875, + "max_features_per_member": 330.0, + "min_members_per_feature": 18.0, + "mean_members_per_feature": 25.68877601623535, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.6710553952491272, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 592, + "metrics": { + "train": { + "rmse": 0.24658122298608637, + "mae": 0.13536591827869415, + "r2": 0.6115511059761047, + "score": -0.24658122298608637 + }, + "val": { + "rmse": 0.2385396806698102, + "mae": 0.13372096419334412, + "r2": 0.6199221611022949, + "score": -0.2385396806698102 + }, + "test": { + "rmse": 0.2407010051283554, + "mae": 0.15062788128852844, + "r2": 0.6040909290313721, + "score": -0.2407010051283554 + } + }, + "time": "0:00:52.144209", + "diversity": { + "train": { + "mean_centered_corr": -0.01851801574230194, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04785211756825447 + }, + "val": { + "mean_centered_corr": -0.019852403551340103, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0691390112042427 + }, + "test": { + "mean_centered_corr": -0.019887704402208328, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08872178196907043 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 1 + ], + "head_selection_time": "0:00:00.015140" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-best-head-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-best-head-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-best-head-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-best-head-evaluation/2/report.json new file mode 100644 index 000000000..895187baa --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-best-head-evaluation/2/report.json @@ -0,0 +1,226 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + }, + "mask_stats": { + "keep_rate_actual": 0.8008609414100647, + "min_features_per_member": 293.0, + "mean_features_per_member": 313.9375, + "max_features_per_member": 331.0, + "min_members_per_feature": 19.0, + "mean_members_per_feature": 25.62755012512207, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.6674885740537666, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1110, + "metrics": { + "train": { + "rmse": 0.22528715287379555, + "mae": 0.12278623133897781, + "r2": 0.6757449507713318, + "score": -0.22528715287379555 + }, + "val": { + "rmse": 0.24181054374010003, + "mae": 0.1348293274641037, + "r2": 0.6094274520874023, + "score": -0.24181054374010003 + }, + "test": { + "rmse": 0.24863506025450496, + "mae": 0.16692666709423065, + "r2": 0.5775606632232666, + "score": -0.24863506025450496 + } + }, + "time": "0:01:06.264634", + "diversity": { + "train": { + "mean_centered_corr": -0.01694483496248722, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0545165091753006 + }, + "val": { + "mean_centered_corr": -0.015533894300460815, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08842652291059494 + }, + "test": { + "mean_centered_corr": -0.016092214733362198, + "mean_pairwise_disagreement": NaN, + "member_std": 0.11407508701086044 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 2 + ], + "head_selection_time": "0:00:00.014901" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/0.toml new file mode 100644 index 000000000..d44bc73bf --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/0.toml @@ -0,0 +1,47 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.8 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/0/report.json new file mode 100644 index 000000000..66077afb4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/0/report.json @@ -0,0 +1,222 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.8038105964660645, + "min_features_per_member": 298.0, + "mean_features_per_member": 315.09375, + "max_features_per_member": 335.0, + "min_members_per_feature": 19.0, + "mean_members_per_feature": 25.721939086914062, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.6715097166219348, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.22081238228152064, + "mae": 0.11986075341701508, + "r2": 0.6884980797767639, + "score": -0.22081238228152064 + }, + "val": { + "rmse": 0.2341279953511937, + "mae": 0.12804140150547028, + "r2": 0.6338509321212769, + "score": -0.2341279953511937 + }, + "test": { + "rmse": 0.23402994505001048, + "mae": 0.14647243916988373, + "r2": 0.6257321834564209, + "score": -0.23402994505001048 + } + }, + "time": "0:00:59.854964", + "diversity": { + "train": { + "mean_centered_corr": -0.017583154141902924, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05003640055656433 + }, + "val": { + "mean_centered_corr": -0.018760371953248978, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07571490854024887 + }, + "test": { + "mean_centered_corr": -0.019624220207333565, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09894119203090668 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/1.toml new file mode 100644 index 000000000..eba42670e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/1.toml @@ -0,0 +1,47 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 1 +head_selection = true + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.8 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 1 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/1/report.json new file mode 100644 index 000000000..2ca6ea33a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/1/report.json @@ -0,0 +1,222 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + }, + "mask_stats": { + "keep_rate_actual": 0.8027742505073547, + "min_features_per_member": 296.0, + "mean_features_per_member": 314.6875, + "max_features_per_member": 330.0, + "min_members_per_feature": 18.0, + "mean_members_per_feature": 25.68877601623535, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.6710553952491272, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 592, + "metrics": { + "train": { + "rmse": 0.24114945435053933, + "mae": 0.12663434445858002, + "r2": 0.6284763813018799, + "score": -0.24114945435053933 + }, + "val": { + "rmse": 0.23621923128975542, + "mae": 0.12754125893115997, + "r2": 0.6272808313369751, + "score": -0.23621923128975542 + }, + "test": { + "rmse": 0.23276193568912745, + "mae": 0.14217989146709442, + "r2": 0.6297768950462341, + "score": -0.23276193568912745 + } + }, + "time": "0:00:52.144209", + "diversity": { + "train": { + "mean_centered_corr": -0.01851801574230194, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04785211756825447 + }, + "val": { + "mean_centered_corr": -0.019852403551340103, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0691390112042427 + }, + "test": { + "mean_centered_corr": -0.019887704402208328, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08872178196907043 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/2.toml new file mode 100644 index 000000000..c8868541f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/2.toml @@ -0,0 +1,47 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 2 +head_selection = true + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.8 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 2 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/2/report.json new file mode 100644 index 000000000..b9db155ac --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/2/report.json @@ -0,0 +1,222 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + }, + "mask_stats": { + "keep_rate_actual": 0.8008609414100647, + "min_features_per_member": 293.0, + "mean_features_per_member": 313.9375, + "max_features_per_member": 331.0, + "min_members_per_feature": 19.0, + "mean_members_per_feature": 25.62755012512207, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.6674885740537666, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1110, + "metrics": { + "train": { + "rmse": 0.21154753458799463, + "mae": 0.11532945930957794, + "r2": 0.7140896916389465, + "score": -0.21154753458799463 + }, + "val": { + "rmse": 0.23605616345434166, + "mae": 0.12597376108169556, + "r2": 0.6277952194213867, + "score": -0.23605616345434166 + }, + "test": { + "rmse": 0.23451195530114213, + "mae": 0.14579595625400543, + "r2": 0.6241888999938965, + "score": -0.23451195530114213 + } + }, + "time": "0:01:06.264634", + "diversity": { + "train": { + "mean_centered_corr": -0.01694483496248722, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0545165091753006 + }, + "val": { + "mean_centered_corr": -0.015533894300460815, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08842652291059494 + }, + "test": { + "mean_centered_corr": -0.016092214733362198, + "mean_pairwise_disagreement": NaN, + "member_std": 0.11407508701086044 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..b613a292e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,235 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.8038105964660645, + "min_features_per_member": 298.0, + "mean_features_per_member": 315.09375, + "max_features_per_member": 335.0, + "min_members_per_feature": 19.0, + "mean_members_per_feature": 25.721939086914062, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.6715097166219348, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.22132344126163775, + "mae": 0.11854095757007599, + "r2": 0.6870545148849487, + "score": -0.22132344126163775 + }, + "val": { + "rmse": 0.23237596587359466, + "mae": 0.1258111149072647, + "r2": 0.639310359954834, + "score": -0.23237596587359466 + }, + "test": { + "rmse": 0.23064696162155762, + "mae": 0.1411736011505127, + "r2": 0.6364743709564209, + "score": -0.23064696162155762 + } + }, + "time": "0:00:59.854964", + "diversity": { + "train": { + "mean_centered_corr": -0.017583154141902924, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05003640055656433 + }, + "val": { + "mean_centered_corr": -0.018760371953248978, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07571490854024887 + }, + "test": { + "mean_centered_corr": -0.019624220207333565, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09894119203090668 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25, + 28, + 2, + 27, + 12, + 24, + 9, + 3, + 21, + 28 + ], + "head_selection_time": "0:00:00.166841" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/1/report.json new file mode 100644 index 000000000..7dee0435b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/1/report.json @@ -0,0 +1,235 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + }, + "mask_stats": { + "keep_rate_actual": 0.8027742505073547, + "min_features_per_member": 296.0, + "mean_features_per_member": 314.6875, + "max_features_per_member": 330.0, + "min_members_per_feature": 18.0, + "mean_members_per_feature": 25.68877601623535, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.6710553952491272, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 592, + "metrics": { + "train": { + "rmse": 0.24172454855386824, + "mae": 0.1285606175661087, + "r2": 0.6267021894454956, + "score": -0.24172454855386824 + }, + "val": { + "rmse": 0.23377133352416926, + "mae": 0.1269364356994629, + "r2": 0.6349656581878662, + "score": -0.23377133352416926 + }, + "test": { + "rmse": 0.23643993065905872, + "mae": 0.152057945728302, + "r2": 0.6179842948913574, + "score": -0.23643993065905872 + } + }, + "time": "0:00:52.144209", + "diversity": { + "train": { + "mean_centered_corr": -0.01851801574230194, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04785211756825447 + }, + "val": { + "mean_centered_corr": -0.019852403551340103, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0691390112042427 + }, + "test": { + "mean_centered_corr": -0.019887704402208328, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08872178196907043 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 1, + 4, + 0, + 6, + 17, + 18, + 30, + 23, + 17, + 4 + ], + "head_selection_time": "0:00:00.159814" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/2/report.json new file mode 100644 index 000000000..1ea10ba61 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/2/report.json @@ -0,0 +1,238 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + }, + "mask_stats": { + "keep_rate_actual": 0.8008609414100647, + "min_features_per_member": 293.0, + "mean_features_per_member": 313.9375, + "max_features_per_member": 331.0, + "min_members_per_feature": 19.0, + "mean_members_per_feature": 25.62755012512207, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.6674885740537666, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1110, + "metrics": { + "train": { + "rmse": 0.21430787098953585, + "mae": 0.11510790884494781, + "r2": 0.7065796852111816, + "score": -0.21430787098953585 + }, + "val": { + "rmse": 0.23350988305065445, + "mae": 0.12613466382026672, + "r2": 0.6357816457748413, + "score": -0.23350988305065445 + }, + "test": { + "rmse": 0.23821200474658916, + "mae": 0.15480704605579376, + "r2": 0.6122365593910217, + "score": -0.23821200474658916 + } + }, + "time": "0:01:06.264634", + "diversity": { + "train": { + "mean_centered_corr": -0.01694483496248722, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0545165091753006 + }, + "val": { + "mean_centered_corr": -0.015533894300460815, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08842652291059494 + }, + "test": { + "mean_centered_corr": -0.016092214733362198, + "mean_pairwise_disagreement": NaN, + "member_std": 0.11407508701086044 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 2, + 22, + 24, + 9, + 20, + 5, + 14, + 22, + 16, + 30, + 24, + 9, + 22 + ], + "head_selection_time": "0:00:00.203418" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..45688715d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-best-head-evaluation/0/report.json @@ -0,0 +1,203 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22423689267064406, + "mae": 0.12748517096042633, + "r2": 0.6787611842155457, + "score": -0.22423689267064406 + }, + "val": { + "rmse": 0.2370483756746748, + "mae": 0.13395926356315613, + "r2": 0.6246596574783325, + "score": -0.2370483756746748 + }, + "test": { + "rmse": 0.23920040609688478, + "mae": 0.14310523867607117, + "r2": 0.6090120077133179, + "score": -0.23920040609688478 + } + }, + "time": "0:01:09.057982", + "diversity": { + "train": { + "mean_centered_corr": -0.019481390714645386, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05119703337550163 + }, + "val": { + "mean_centered_corr": -0.01973012462258339, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07138136774301529 + }, + "test": { + "mean_centered_corr": -0.016405168920755386, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09870413690805435 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19 + ], + "head_selection_time": "0:00:00.015930" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-best-head-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-best-head-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-best-head-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-best-head-evaluation/1/report.json new file mode 100644 index 000000000..e42d7cf5c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-best-head-evaluation/1/report.json @@ -0,0 +1,203 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.227632937078759, + "mae": 0.123143769800663, + "r2": 0.6689572334289551, + "score": -0.227632937078759 + }, + "val": { + "rmse": 0.23898563084143007, + "mae": 0.13369078934192657, + "r2": 0.618499755859375, + "score": -0.23898563084143007 + }, + "test": { + "rmse": 0.23816985512587505, + "mae": 0.14563022553920746, + "r2": 0.6123737096786499, + "score": -0.23816985512587505 + } + }, + "time": "0:01:10.237258", + "diversity": { + "train": { + "mean_centered_corr": -0.018491873517632484, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04902740567922592 + }, + "val": { + "mean_centered_corr": -0.017897142097353935, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07918199896812439 + }, + "test": { + "mean_centered_corr": -0.016073623672127724, + "mean_pairwise_disagreement": NaN, + "member_std": 0.1072448268532753 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 1 + ], + "head_selection_time": "0:00:00.015321" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-best-head-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-best-head-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-best-head-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-best-head-evaluation/2/report.json new file mode 100644 index 000000000..53d5ea3b7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-best-head-evaluation/2/report.json @@ -0,0 +1,203 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2373874975071392, + "mae": 0.13008663058280945, + "r2": 0.6399775743484497, + "score": -0.2373874975071392 + }, + "val": { + "rmse": 0.2381416834471136, + "mae": 0.13739991188049316, + "r2": 0.6211894750595093, + "score": -0.2381416834471136 + }, + "test": { + "rmse": 0.2498549025398126, + "mae": 0.1722332090139389, + "r2": 0.573405385017395, + "score": -0.2498549025398126 + } + }, + "time": "0:01:04.094967", + "diversity": { + "train": { + "mean_centered_corr": -0.019239937886595726, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04747721925377846 + }, + "val": { + "mean_centered_corr": -0.019888369366526604, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06191844120621681 + }, + "test": { + "mean_centered_corr": -0.019547125324606895, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07305453717708588 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015066" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-evaluation/0.toml new file mode 100644 index 000000000..44f3d87b5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-evaluation/0.toml @@ -0,0 +1,41 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-evaluation/0/report.json new file mode 100644 index 000000000..475cad836 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-evaluation/0/report.json @@ -0,0 +1,199 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22159317613989218, + "mae": 0.12286020070314407, + "r2": 0.6862912178039551, + "score": -0.22159317613989218 + }, + "val": { + "rmse": 0.2354682854519358, + "mae": 0.13157758116722107, + "r2": 0.6296467781066895, + "score": -0.2354682854519358 + }, + "test": { + "rmse": 0.24014251007009368, + "mae": 0.15741941332817078, + "r2": 0.6059260964393616, + "score": -0.24014251007009368 + } + }, + "time": "0:01:09.057982", + "diversity": { + "train": { + "mean_centered_corr": -0.019481390714645386, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05119703337550163 + }, + "val": { + "mean_centered_corr": -0.01973012462258339, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07138136774301529 + }, + "test": { + "mean_centered_corr": -0.016405168920755386, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09870413690805435 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-evaluation/1.toml new file mode 100644 index 000000000..a9dd39142 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-evaluation/1.toml @@ -0,0 +1,41 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 1 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-evaluation/1/report.json new file mode 100644 index 000000000..cc70515b7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-evaluation/1/report.json @@ -0,0 +1,199 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2219400179657564, + "mae": 0.11737057566642761, + "r2": 0.6853084564208984, + "score": -0.2219400179657564 + }, + "val": { + "rmse": 0.23689852205208367, + "mae": 0.12702235579490662, + "r2": 0.6251341104507446, + "score": -0.23689852205208367 + }, + "test": { + "rmse": 0.23132638249744272, + "mae": 0.13775070011615753, + "r2": 0.6343294978141785, + "score": -0.23132638249744272 + } + }, + "time": "0:01:10.237258", + "diversity": { + "train": { + "mean_centered_corr": -0.018491873517632484, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04902740567922592 + }, + "val": { + "mean_centered_corr": -0.017897142097353935, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07918199896812439 + }, + "test": { + "mean_centered_corr": -0.016073623672127724, + "mean_pairwise_disagreement": NaN, + "member_std": 0.1072448268532753 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-evaluation/2.toml new file mode 100644 index 000000000..05b5f4603 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-evaluation/2.toml @@ -0,0 +1,41 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 2 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-evaluation/2/report.json new file mode 100644 index 000000000..c8ac09e7e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-evaluation/2/report.json @@ -0,0 +1,199 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2320526781045563, + "mae": 0.12807966768741608, + "r2": 0.6559773683547974, + "score": -0.2320526781045563 + }, + "val": { + "rmse": 0.2359145620703149, + "mae": 0.1308077573776245, + "r2": 0.6282416582107544, + "score": -0.2359145620703149 + }, + "test": { + "rmse": 0.2332252993496603, + "mae": 0.14604337513446808, + "r2": 0.6283013820648193, + "score": -0.2332252993496603 + } + }, + "time": "0:01:04.094967", + "diversity": { + "train": { + "mean_centered_corr": -0.019239937886595726, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04747721925377846 + }, + "val": { + "mean_centered_corr": -0.019888369366526604, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06191844120621681 + }, + "test": { + "mean_centered_corr": -0.019547125324606895, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07305453717708588 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..42d104b71 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,206 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22248795490129444, + "mae": 0.1261499673128128, + "r2": 0.6837526559829712, + "score": -0.22248795490129444 + }, + "val": { + "rmse": 0.2328613038082034, + "mae": 0.12715381383895874, + "r2": 0.6378021240234375, + "score": -0.2328613038082034 + }, + "test": { + "rmse": 0.23498425768312764, + "mae": 0.14776451885700226, + "r2": 0.6226736307144165, + "score": -0.23498425768312764 + } + }, + "time": "0:01:09.057982", + "diversity": { + "train": { + "mean_centered_corr": -0.019481390714645386, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05119703337550163 + }, + "val": { + "mean_centered_corr": -0.01973012462258339, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07138136774301529 + }, + "test": { + "mean_centered_corr": -0.016405168920755386, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09870413690805435 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19, + 21, + 4, + 24 + ], + "head_selection_time": "0:00:00.076523" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/1/report.json new file mode 100644 index 000000000..36bb022b8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/1/report.json @@ -0,0 +1,206 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2235184256182525, + "mae": 0.12183725833892822, + "r2": 0.6808164119720459, + "score": -0.2235184256182525 + }, + "val": { + "rmse": 0.23460022912632159, + "mae": 0.12736009061336517, + "r2": 0.6323723793029785, + "score": -0.23460022912632159 + }, + "test": { + "rmse": 0.23076016380764466, + "mae": 0.13752052187919617, + "r2": 0.6361174583435059, + "score": -0.23076016380764466 + } + }, + "time": "0:01:10.237258", + "diversity": { + "train": { + "mean_centered_corr": -0.018491873517632484, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04902740567922592 + }, + "val": { + "mean_centered_corr": -0.017897142097353935, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07918199896812439 + }, + "test": { + "mean_centered_corr": -0.016073623672127724, + "mean_pairwise_disagreement": NaN, + "member_std": 0.1072448268532753 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 1, + 19, + 28, + 6 + ], + "head_selection_time": "0:00:00.076399" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/2/report.json new file mode 100644 index 000000000..234b01fa9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/2/report.json @@ -0,0 +1,215 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2323423137248424, + "mae": 0.12613606452941895, + "r2": 0.6551179885864258, + "score": -0.2323423137248424 + }, + "val": { + "rmse": 0.23410085698681354, + "mae": 0.12800754606723785, + "r2": 0.633935809135437, + "score": -0.23410085698681354 + }, + "test": { + "rmse": 0.23135944978887066, + "mae": 0.14349235594272614, + "r2": 0.6342249512672424, + "score": -0.23135944978887066 + } + }, + "time": "0:01:04.094967", + "diversity": { + "train": { + "mean_centered_corr": -0.019239937886595726, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04747721925377846 + }, + "val": { + "mean_centered_corr": -0.019888369366526604, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06191844120621681 + }, + "test": { + "mean_centered_corr": -0.019547125324606895, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07305453717708588 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 30, + 5, + 12, + 13, + 30, + 12, + 15, + 9, + 11, + 22, + 2, + 23 + ], + "head_selection_time": "0:00:00.202277" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/manifest_baseline_fidelity.txt b/tabm_integrated/paper/exp/integrated/manifest_baseline_fidelity.txt new file mode 100644 index 000000000..fb976ae58 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/manifest_baseline_fidelity.txt @@ -0,0 +1,5 @@ +exp/integrated/baseline_fidelity/sberbank-housing/baseline_plr-evaluation/0.toml +exp/integrated/baseline_fidelity/ecom-offers/baseline_plr-evaluation/0.toml +exp/integrated/baseline_fidelity/homesite-insurance/baseline_plr-evaluation/0.toml +exp/integrated/baseline_fidelity/cooking-time/baseline_plr-evaluation/0.toml +exp/integrated/baseline_fidelity/delivery-eta/baseline_plr-evaluation/0.toml diff --git a/tabm_integrated/paper/exp/integrated/manifest_delivery_rescue_combined.txt b/tabm_integrated/paper/exp/integrated/manifest_delivery_rescue_combined.txt new file mode 100644 index 000000000..c798e4296 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/manifest_delivery_rescue_combined.txt @@ -0,0 +1,4 @@ +exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam-evaluation/0.toml +exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_cf-evaluation/0.toml +exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_esam_cf-evaluation/0.toml +exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam_cf-evaluation/0.toml diff --git a/tabm_integrated/paper/exp/integrated/manifest_delivery_rescue_final.txt b/tabm_integrated/paper/exp/integrated/manifest_delivery_rescue_final.txt new file mode 100644 index 000000000..f52fdce08 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/manifest_delivery_rescue_final.txt @@ -0,0 +1,15 @@ +exp/integrated/final/delivery-eta/best_rla_only-evaluation/0.toml +exp/integrated/final/delivery-eta/best_rla_only-evaluation/1.toml +exp/integrated/final/delivery-eta/best_rla_only-evaluation/2.toml +exp/integrated/final/delivery-eta/best_esam_only-evaluation/0.toml +exp/integrated/final/delivery-eta/best_esam_only-evaluation/1.toml +exp/integrated/final/delivery-eta/best_esam_only-evaluation/2.toml +exp/integrated/final/delivery-eta/best_mfb_only-evaluation/0.toml +exp/integrated/final/delivery-eta/best_mfb_only-evaluation/1.toml +exp/integrated/final/delivery-eta/best_mfb_only-evaluation/2.toml +exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/0.toml +exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/1.toml +exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/2.toml +exp/integrated/final/delivery-eta/best_combined-evaluation/0.toml +exp/integrated/final/delivery-eta/best_combined-evaluation/1.toml +exp/integrated/final/delivery-eta/best_combined-evaluation/2.toml diff --git a/tabm_integrated/paper/exp/integrated/manifest_delivery_rescue_individual.txt b/tabm_integrated/paper/exp/integrated/manifest_delivery_rescue_individual.txt new file mode 100644 index 000000000..bafc9df84 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/manifest_delivery_rescue_individual.txt @@ -0,0 +1,62 @@ +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0p5-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0p5-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0p5-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0p5-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0p5-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0p5-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0p5-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0p5-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0p5-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0p5-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0p5-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0p5-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0p5-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0p5-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0p5-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0p5-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0p5-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0p5-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adaptert-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adapterf-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adaptert-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adapterf-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adaptert-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adapterf-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adaptert-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adapterf-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adaptert-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adapterf-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adaptert-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adapterf-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_numerical_only-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_per_member-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_numerical_only-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_per_member-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_numerical_only-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_per_member-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_numerical_only-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_per_member-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_consensus_raw-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_softmax-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_consensus_raw-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_softmax-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_consensus_raw-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_softmax-evaluation/0.toml diff --git a/tabm_integrated/paper/exp/integrated/manifest_final.txt b/tabm_integrated/paper/exp/integrated/manifest_final.txt new file mode 100644 index 000000000..2a09cdc7e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/manifest_final.txt @@ -0,0 +1,90 @@ +exp/integrated/final/sberbank-housing/baseline_plr-evaluation/0.toml +exp/integrated/final/sberbank-housing/baseline_plr-evaluation/1.toml +exp/integrated/final/sberbank-housing/baseline_plr-evaluation/2.toml +exp/integrated/final/sberbank-housing/best_rla_only-evaluation/0.toml +exp/integrated/final/sberbank-housing/best_rla_only-evaluation/1.toml +exp/integrated/final/sberbank-housing/best_rla_only-evaluation/2.toml +exp/integrated/final/sberbank-housing/best_esam_only-evaluation/0.toml +exp/integrated/final/sberbank-housing/best_esam_only-evaluation/1.toml +exp/integrated/final/sberbank-housing/best_esam_only-evaluation/2.toml +exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/0.toml +exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/1.toml +exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/2.toml +exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/0.toml +exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/1.toml +exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/2.toml +exp/integrated/final/sberbank-housing/best_combined-evaluation/0.toml +exp/integrated/final/sberbank-housing/best_combined-evaluation/1.toml +exp/integrated/final/sberbank-housing/best_combined-evaluation/2.toml +exp/integrated/final/ecom-offers/baseline_plr-evaluation/0.toml +exp/integrated/final/ecom-offers/baseline_plr-evaluation/1.toml +exp/integrated/final/ecom-offers/baseline_plr-evaluation/2.toml +exp/integrated/final/ecom-offers/best_rla_only-evaluation/0.toml +exp/integrated/final/ecom-offers/best_rla_only-evaluation/1.toml +exp/integrated/final/ecom-offers/best_rla_only-evaluation/2.toml +exp/integrated/final/ecom-offers/best_esam_only-evaluation/0.toml +exp/integrated/final/ecom-offers/best_esam_only-evaluation/1.toml +exp/integrated/final/ecom-offers/best_esam_only-evaluation/2.toml +exp/integrated/final/ecom-offers/best_mfb_only-evaluation/0.toml +exp/integrated/final/ecom-offers/best_mfb_only-evaluation/1.toml +exp/integrated/final/ecom-offers/best_mfb_only-evaluation/2.toml +exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/0.toml +exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/1.toml +exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/2.toml +exp/integrated/final/ecom-offers/best_combined-evaluation/0.toml +exp/integrated/final/ecom-offers/best_combined-evaluation/1.toml +exp/integrated/final/ecom-offers/best_combined-evaluation/2.toml +exp/integrated/final/homesite-insurance/baseline_plr-evaluation/0.toml +exp/integrated/final/homesite-insurance/baseline_plr-evaluation/1.toml +exp/integrated/final/homesite-insurance/baseline_plr-evaluation/2.toml +exp/integrated/final/homesite-insurance/best_rla_only-evaluation/0.toml +exp/integrated/final/homesite-insurance/best_rla_only-evaluation/1.toml +exp/integrated/final/homesite-insurance/best_rla_only-evaluation/2.toml +exp/integrated/final/homesite-insurance/best_esam_only-evaluation/0.toml +exp/integrated/final/homesite-insurance/best_esam_only-evaluation/1.toml +exp/integrated/final/homesite-insurance/best_esam_only-evaluation/2.toml +exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/0.toml +exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/1.toml +exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/2.toml +exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/0.toml +exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/1.toml +exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/2.toml +exp/integrated/final/homesite-insurance/best_combined-evaluation/0.toml +exp/integrated/final/homesite-insurance/best_combined-evaluation/1.toml +exp/integrated/final/homesite-insurance/best_combined-evaluation/2.toml +exp/integrated/final/cooking-time/baseline_plr-evaluation/0.toml +exp/integrated/final/cooking-time/baseline_plr-evaluation/1.toml +exp/integrated/final/cooking-time/baseline_plr-evaluation/2.toml +exp/integrated/final/cooking-time/best_rla_only-evaluation/0.toml +exp/integrated/final/cooking-time/best_rla_only-evaluation/1.toml +exp/integrated/final/cooking-time/best_rla_only-evaluation/2.toml +exp/integrated/final/cooking-time/best_esam_only-evaluation/0.toml +exp/integrated/final/cooking-time/best_esam_only-evaluation/1.toml +exp/integrated/final/cooking-time/best_esam_only-evaluation/2.toml +exp/integrated/final/cooking-time/best_mfb_only-evaluation/0.toml +exp/integrated/final/cooking-time/best_mfb_only-evaluation/1.toml +exp/integrated/final/cooking-time/best_mfb_only-evaluation/2.toml +exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/0.toml +exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/1.toml +exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/2.toml +exp/integrated/final/cooking-time/best_combined-evaluation/0.toml +exp/integrated/final/cooking-time/best_combined-evaluation/1.toml +exp/integrated/final/cooking-time/best_combined-evaluation/2.toml +exp/integrated/final/delivery-eta/baseline_plr-evaluation/0.toml +exp/integrated/final/delivery-eta/baseline_plr-evaluation/1.toml +exp/integrated/final/delivery-eta/baseline_plr-evaluation/2.toml +exp/integrated/final/delivery-eta/best_rla_only-evaluation/0.toml +exp/integrated/final/delivery-eta/best_rla_only-evaluation/1.toml +exp/integrated/final/delivery-eta/best_rla_only-evaluation/2.toml +exp/integrated/final/delivery-eta/best_esam_only-evaluation/0.toml +exp/integrated/final/delivery-eta/best_esam_only-evaluation/1.toml +exp/integrated/final/delivery-eta/best_esam_only-evaluation/2.toml +exp/integrated/final/delivery-eta/best_mfb_only-evaluation/0.toml +exp/integrated/final/delivery-eta/best_mfb_only-evaluation/1.toml +exp/integrated/final/delivery-eta/best_mfb_only-evaluation/2.toml +exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/0.toml +exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/1.toml +exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/2.toml +exp/integrated/final/delivery-eta/best_combined-evaluation/0.toml +exp/integrated/final/delivery-eta/best_combined-evaluation/1.toml +exp/integrated/final/delivery-eta/best_combined-evaluation/2.toml diff --git a/tabm_integrated/paper/exp/integrated/manifest_sberbank_combined_rescue_final.txt b/tabm_integrated/paper/exp/integrated/manifest_sberbank_combined_rescue_final.txt new file mode 100644 index 000000000..7a579dfa0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/manifest_sberbank_combined_rescue_final.txt @@ -0,0 +1,6 @@ +exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/1.toml +exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/2.toml +exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/1.toml +exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/2.toml diff --git a/tabm_integrated/paper/exp/integrated/manifest_sberbank_combined_rescue_seed0.txt b/tabm_integrated/paper/exp/integrated/manifest_sberbank_combined_rescue_seed0.txt new file mode 100644 index 000000000..59f65e1a3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/manifest_sberbank_combined_rescue_seed0.txt @@ -0,0 +1,177 @@ +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p02-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p02-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p02-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p02-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p02-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p02-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p02-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p02-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p02-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p02-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p02-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p02-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p00025-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0025-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p00025-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0025-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p00025-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0025-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p00025-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0025-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p00025-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0025-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p00025-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0025-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p00025-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0025-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p00025-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0025-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p00025-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0025-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p00025-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0025-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p00025-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0025-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p00025-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0025-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p01-evaluation/0.toml diff --git a/tabm_integrated/paper/exp/integrated/manifest_smoke.txt b/tabm_integrated/paper/exp/integrated/manifest_smoke.txt new file mode 100644 index 000000000..c28cfbdc2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/manifest_smoke.txt @@ -0,0 +1,30 @@ +exp/integrated/smoke/sberbank-housing/baseline_plr-evaluation/0.toml +exp/integrated/smoke/sberbank-housing/rla_only-evaluation/0.toml +exp/integrated/smoke/sberbank-housing/esam_only-evaluation/0.toml +exp/integrated/smoke/sberbank-housing/mfb_only-evaluation/0.toml +exp/integrated/smoke/sberbank-housing/cf_fisd_only-evaluation/0.toml +exp/integrated/smoke/sberbank-housing/all_four_combined-evaluation/0.toml +exp/integrated/smoke/ecom-offers/baseline_plr-evaluation/0.toml +exp/integrated/smoke/ecom-offers/rla_only-evaluation/0.toml +exp/integrated/smoke/ecom-offers/esam_only-evaluation/0.toml +exp/integrated/smoke/ecom-offers/mfb_only-evaluation/0.toml +exp/integrated/smoke/ecom-offers/cf_fisd_only-evaluation/0.toml +exp/integrated/smoke/ecom-offers/all_four_combined-evaluation/0.toml +exp/integrated/smoke/homesite-insurance/baseline_plr-evaluation/0.toml +exp/integrated/smoke/homesite-insurance/rla_only-evaluation/0.toml +exp/integrated/smoke/homesite-insurance/esam_only-evaluation/0.toml +exp/integrated/smoke/homesite-insurance/mfb_only-evaluation/0.toml +exp/integrated/smoke/homesite-insurance/cf_fisd_only-evaluation/0.toml +exp/integrated/smoke/homesite-insurance/all_four_combined-evaluation/0.toml +exp/integrated/smoke/cooking-time/baseline_plr-evaluation/0.toml +exp/integrated/smoke/cooking-time/rla_only-evaluation/0.toml +exp/integrated/smoke/cooking-time/esam_only-evaluation/0.toml +exp/integrated/smoke/cooking-time/mfb_only-evaluation/0.toml +exp/integrated/smoke/cooking-time/cf_fisd_only-evaluation/0.toml +exp/integrated/smoke/cooking-time/all_four_combined-evaluation/0.toml +exp/integrated/smoke/delivery-eta/baseline_plr-evaluation/0.toml +exp/integrated/smoke/delivery-eta/rla_only-evaluation/0.toml +exp/integrated/smoke/delivery-eta/esam_only-evaluation/0.toml +exp/integrated/smoke/delivery-eta/mfb_only-evaluation/0.toml +exp/integrated/smoke/delivery-eta/cf_fisd_only-evaluation/0.toml +exp/integrated/smoke/delivery-eta/all_four_combined-evaluation/0.toml diff --git a/tabm_integrated/paper/exp/integrated/manifest_sweeps.txt b/tabm_integrated/paper/exp/integrated/manifest_sweeps.txt new file mode 100644 index 000000000..54d1eed59 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/manifest_sweeps.txt @@ -0,0 +1,155 @@ +exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.001-evaluation/0.toml +exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.0001-evaluation/0.toml +exp/integrated/sweeps/sberbank-housing/rla_rank2_noise1e-05-evaluation/0.toml +exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.001-evaluation/0.toml +exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.0001-evaluation/0.toml +exp/integrated/sweeps/sberbank-housing/rla_rank4_noise1e-05-evaluation/0.toml +exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.001-evaluation/0.toml +exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.0001-evaluation/0.toml +exp/integrated/sweeps/sberbank-housing/rla_rank8_noise1e-05-evaluation/0.toml +exp/integrated/sweeps/sberbank-housing/esam_rho0.001-evaluation/0.toml +exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-evaluation/0.toml +exp/integrated/sweeps/sberbank-housing/esam_rho0.005-evaluation/0.toml +exp/integrated/sweeps/sberbank-housing/esam_rho0.01-evaluation/0.toml +exp/integrated/sweeps/sberbank-housing/mfb_keep0.7-evaluation/0.toml +exp/integrated/sweeps/sberbank-housing/mfb_keep0.8-evaluation/0.toml +exp/integrated/sweeps/sberbank-housing/mfb_keep0.9-evaluation/0.toml +exp/integrated/sweeps/sberbank-housing/mfb_keep0.95-evaluation/0.toml +exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.05-evaluation/0.toml +exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.1-evaluation/0.toml +exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.2-evaluation/0.toml +exp/integrated/sweeps/sberbank-housing/rla_esam-evaluation/0.toml +exp/integrated/sweeps/sberbank-housing/rla_mfb-evaluation/0.toml +exp/integrated/sweeps/sberbank-housing/rla_cf_fisd-evaluation/0.toml +exp/integrated/sweeps/sberbank-housing/esam_mfb-evaluation/0.toml +exp/integrated/sweeps/sberbank-housing/esam_cf_fisd-evaluation/0.toml +exp/integrated/sweeps/sberbank-housing/mfb_cf_fisd-evaluation/0.toml +exp/integrated/sweeps/sberbank-housing/all_four_combined-evaluation/0.toml +exp/integrated/sweeps/sberbank-housing/all_minus_rla-evaluation/0.toml +exp/integrated/sweeps/sberbank-housing/all_minus_esam-evaluation/0.toml +exp/integrated/sweeps/sberbank-housing/all_minus_mfb-evaluation/0.toml +exp/integrated/sweeps/sberbank-housing/all_minus_cf_fisd-evaluation/0.toml +exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.001-evaluation/0.toml +exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.0001-evaluation/0.toml +exp/integrated/sweeps/ecom-offers/rla_rank2_noise1e-05-evaluation/0.toml +exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.001-evaluation/0.toml +exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.0001-evaluation/0.toml +exp/integrated/sweeps/ecom-offers/rla_rank4_noise1e-05-evaluation/0.toml +exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.001-evaluation/0.toml +exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.0001-evaluation/0.toml +exp/integrated/sweeps/ecom-offers/rla_rank8_noise1e-05-evaluation/0.toml +exp/integrated/sweeps/ecom-offers/esam_rho0.001-evaluation/0.toml +exp/integrated/sweeps/ecom-offers/esam_rho0.0025-evaluation/0.toml +exp/integrated/sweeps/ecom-offers/esam_rho0.005-evaluation/0.toml +exp/integrated/sweeps/ecom-offers/esam_rho0.01-evaluation/0.toml +exp/integrated/sweeps/ecom-offers/mfb_keep0.7-evaluation/0.toml +exp/integrated/sweeps/ecom-offers/mfb_keep0.8-evaluation/0.toml +exp/integrated/sweeps/ecom-offers/mfb_keep0.9-evaluation/0.toml +exp/integrated/sweeps/ecom-offers/mfb_keep0.95-evaluation/0.toml +exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.05-evaluation/0.toml +exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.1-evaluation/0.toml +exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.2-evaluation/0.toml +exp/integrated/sweeps/ecom-offers/rla_esam-evaluation/0.toml +exp/integrated/sweeps/ecom-offers/rla_mfb-evaluation/0.toml +exp/integrated/sweeps/ecom-offers/rla_cf_fisd-evaluation/0.toml +exp/integrated/sweeps/ecom-offers/esam_mfb-evaluation/0.toml +exp/integrated/sweeps/ecom-offers/esam_cf_fisd-evaluation/0.toml +exp/integrated/sweeps/ecom-offers/mfb_cf_fisd-evaluation/0.toml +exp/integrated/sweeps/ecom-offers/all_four_combined-evaluation/0.toml +exp/integrated/sweeps/ecom-offers/all_minus_rla-evaluation/0.toml +exp/integrated/sweeps/ecom-offers/all_minus_esam-evaluation/0.toml +exp/integrated/sweeps/ecom-offers/all_minus_mfb-evaluation/0.toml +exp/integrated/sweeps/ecom-offers/all_minus_cf_fisd-evaluation/0.toml +exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.001-evaluation/0.toml +exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.0001-evaluation/0.toml +exp/integrated/sweeps/homesite-insurance/rla_rank2_noise1e-05-evaluation/0.toml +exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.001-evaluation/0.toml +exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.0001-evaluation/0.toml +exp/integrated/sweeps/homesite-insurance/rla_rank4_noise1e-05-evaluation/0.toml +exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.001-evaluation/0.toml +exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.0001-evaluation/0.toml +exp/integrated/sweeps/homesite-insurance/rla_rank8_noise1e-05-evaluation/0.toml +exp/integrated/sweeps/homesite-insurance/esam_rho0.001-evaluation/0.toml +exp/integrated/sweeps/homesite-insurance/esam_rho0.0025-evaluation/0.toml +exp/integrated/sweeps/homesite-insurance/esam_rho0.005-evaluation/0.toml +exp/integrated/sweeps/homesite-insurance/esam_rho0.01-evaluation/0.toml +exp/integrated/sweeps/homesite-insurance/mfb_keep0.7-evaluation/0.toml +exp/integrated/sweeps/homesite-insurance/mfb_keep0.8-evaluation/0.toml +exp/integrated/sweeps/homesite-insurance/mfb_keep0.9-evaluation/0.toml +exp/integrated/sweeps/homesite-insurance/mfb_keep0.95-evaluation/0.toml +exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.05-evaluation/0.toml +exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.1-evaluation/0.toml +exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.2-evaluation/0.toml +exp/integrated/sweeps/homesite-insurance/rla_esam-evaluation/0.toml +exp/integrated/sweeps/homesite-insurance/rla_mfb-evaluation/0.toml +exp/integrated/sweeps/homesite-insurance/rla_cf_fisd-evaluation/0.toml +exp/integrated/sweeps/homesite-insurance/esam_mfb-evaluation/0.toml +exp/integrated/sweeps/homesite-insurance/esam_cf_fisd-evaluation/0.toml +exp/integrated/sweeps/homesite-insurance/mfb_cf_fisd-evaluation/0.toml +exp/integrated/sweeps/homesite-insurance/all_four_combined-evaluation/0.toml +exp/integrated/sweeps/homesite-insurance/all_minus_rla-evaluation/0.toml +exp/integrated/sweeps/homesite-insurance/all_minus_esam-evaluation/0.toml +exp/integrated/sweeps/homesite-insurance/all_minus_mfb-evaluation/0.toml +exp/integrated/sweeps/homesite-insurance/all_minus_cf_fisd-evaluation/0.toml +exp/integrated/sweeps/cooking-time/rla_rank2_noise0.001-evaluation/0.toml +exp/integrated/sweeps/cooking-time/rla_rank2_noise0.0001-evaluation/0.toml +exp/integrated/sweeps/cooking-time/rla_rank2_noise1e-05-evaluation/0.toml +exp/integrated/sweeps/cooking-time/rla_rank4_noise0.001-evaluation/0.toml +exp/integrated/sweeps/cooking-time/rla_rank4_noise0.0001-evaluation/0.toml +exp/integrated/sweeps/cooking-time/rla_rank4_noise1e-05-evaluation/0.toml +exp/integrated/sweeps/cooking-time/rla_rank8_noise0.001-evaluation/0.toml +exp/integrated/sweeps/cooking-time/rla_rank8_noise0.0001-evaluation/0.toml +exp/integrated/sweeps/cooking-time/rla_rank8_noise1e-05-evaluation/0.toml +exp/integrated/sweeps/cooking-time/esam_rho0.001-evaluation/0.toml +exp/integrated/sweeps/cooking-time/esam_rho0.0025-evaluation/0.toml +exp/integrated/sweeps/cooking-time/esam_rho0.005-evaluation/0.toml +exp/integrated/sweeps/cooking-time/esam_rho0.01-evaluation/0.toml +exp/integrated/sweeps/cooking-time/mfb_keep0.7-evaluation/0.toml +exp/integrated/sweeps/cooking-time/mfb_keep0.8-evaluation/0.toml +exp/integrated/sweeps/cooking-time/mfb_keep0.9-evaluation/0.toml +exp/integrated/sweeps/cooking-time/mfb_keep0.95-evaluation/0.toml +exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.05-evaluation/0.toml +exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.1-evaluation/0.toml +exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.2-evaluation/0.toml +exp/integrated/sweeps/cooking-time/rla_esam-evaluation/0.toml +exp/integrated/sweeps/cooking-time/rla_mfb-evaluation/0.toml +exp/integrated/sweeps/cooking-time/rla_cf_fisd-evaluation/0.toml +exp/integrated/sweeps/cooking-time/esam_mfb-evaluation/0.toml +exp/integrated/sweeps/cooking-time/esam_cf_fisd-evaluation/0.toml +exp/integrated/sweeps/cooking-time/mfb_cf_fisd-evaluation/0.toml +exp/integrated/sweeps/cooking-time/all_four_combined-evaluation/0.toml +exp/integrated/sweeps/cooking-time/all_minus_rla-evaluation/0.toml +exp/integrated/sweeps/cooking-time/all_minus_esam-evaluation/0.toml +exp/integrated/sweeps/cooking-time/all_minus_mfb-evaluation/0.toml +exp/integrated/sweeps/cooking-time/all_minus_cf_fisd-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.001-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.0001-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/rla_rank2_noise1e-05-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.001-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.0001-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/rla_rank4_noise1e-05-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.001-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.0001-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/rla_rank8_noise1e-05-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/esam_rho0.001-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/esam_rho0.0025-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/esam_rho0.005-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/esam_rho0.01-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/mfb_keep0.7-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/mfb_keep0.8-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/mfb_keep0.9-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/mfb_keep0.95-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.05-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.1-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.2-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/rla_esam-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/rla_mfb-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/rla_cf_fisd-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/esam_mfb-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/esam_cf_fisd-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/mfb_cf_fisd-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/all_four_combined-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/all_minus_rla-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/all_minus_esam-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/all_minus_mfb-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/all_minus_cf_fisd-evaluation/0.toml diff --git a/tabm_integrated/paper/exp/integrated/manifest_sweeps_remaining.txt b/tabm_integrated/paper/exp/integrated/manifest_sweeps_remaining.txt new file mode 100644 index 000000000..04872af88 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/manifest_sweeps_remaining.txt @@ -0,0 +1,18 @@ +exp/integrated/sweeps/delivery-eta/mfb_keep0.7-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/mfb_keep0.8-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/mfb_keep0.9-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/mfb_keep0.95-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.05-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.1-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.2-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/rla_esam-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/rla_mfb-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/rla_cf_fisd-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/esam_mfb-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/esam_cf_fisd-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/mfb_cf_fisd-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/all_four_combined-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/all_minus_rla-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/all_minus_esam-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/all_minus_mfb-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/all_minus_cf_fisd-evaluation/0.toml diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_esam_cf-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_esam_cf-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_esam_cf-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_esam_cf-best-head-evaluation/0/report.json new file mode 100644 index 000000000..7b026e464 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_esam_cf-best-head-evaluation/0/report.json @@ -0,0 +1,683 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_esam_cf-evaluation/_esam_diagnostics/deliv_comb_esam_cf-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2730, + "metrics": { + "train": { + "rmse": 0.5056157353876229, + "mae": 0.3998609483242035, + "r2": 0.5208324193954468, + "score": -0.5056157353876229 + }, + "val": { + "rmse": 0.5596782586864594, + "mae": 0.44516822695732117, + "r2": 0.4093165993690491, + "score": -0.5596782586864594 + }, + "test": { + "rmse": 0.5565908577070521, + "mae": 0.45004433393478394, + "r2": 0.42272716760635376, + "score": -0.5565908577070521 + } + }, + "time": "0:14:24.819984", + "diversity": { + "train": { + "mean_centered_corr": -0.020204994827508926, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07431785017251968 + }, + "val": { + "mean_centered_corr": -0.020286401733756065, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07910650968551636 + }, + "test": { + "mean_centered_corr": -0.02015332505106926, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07623301446437836 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 24 + ], + "head_selection_time": "0:00:00.023606" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_esam_cf-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_esam_cf-evaluation/0.toml new file mode 100644 index 000000000..a0e3cdaa7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_esam_cf-evaluation/0.toml @@ -0,0 +1,161 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.001 +esam_eps = 1e-12 +esam_adapter_only = false +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 8 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "delivery-eta" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] +start_epoch = 16 +start_fraction = 0.5 +mode = "consensus_raw" + +[cf_fisd.member_groups] +xgb = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] +lgbm = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] +cat = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_esam_cf-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_esam_cf-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_esam_cf-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_esam_cf-evaluation/0/report.json new file mode 100644 index 000000000..f3330ce7e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_esam_cf-evaluation/0/report.json @@ -0,0 +1,679 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_esam_cf-evaluation/_esam_diagnostics/deliv_comb_esam_cf-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2730, + "metrics": { + "train": { + "rmse": 0.4975037099949126, + "mae": 0.39285650849342346, + "r2": 0.5360844135284424, + "score": -0.4975037099949126 + }, + "val": { + "rmse": 0.5558269510040955, + "mae": 0.44211816787719727, + "r2": 0.4174179434776306, + "score": -0.5558269510040955 + }, + "test": { + "rmse": 0.5527269612276788, + "mae": 0.44693246483802795, + "r2": 0.4307142496109009, + "score": -0.5527269612276788 + } + }, + "time": "0:14:24.819984", + "diversity": { + "train": { + "mean_centered_corr": -0.020204994827508926, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07431785017251968 + }, + "val": { + "mean_centered_corr": -0.020286401733756065, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07910650968551636 + }, + "test": { + "mean_centered_corr": -0.02015332505106926, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07623301446437836 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_esam_cf-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_esam_cf-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_esam_cf-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_esam_cf-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..4367ef428 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_esam_cf-greedy-heads-evaluation/0/report.json @@ -0,0 +1,706 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_esam_cf-evaluation/_esam_diagnostics/deliv_comb_esam_cf-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2730, + "metrics": { + "train": { + "rmse": 0.4981544378477528, + "mae": 0.3936895430088043, + "r2": 0.5348700881004333, + "score": -0.4981544378477528 + }, + "val": { + "rmse": 0.555452544543325, + "mae": 0.44208067655563354, + "r2": 0.4182025194168091, + "score": -0.555452544543325 + }, + "test": { + "rmse": 0.5532771704027001, + "mae": 0.44795727729797363, + "r2": 0.42958033084869385, + "score": -0.5532771704027001 + } + }, + "time": "0:14:24.819984", + "diversity": { + "train": { + "mean_centered_corr": -0.020204994827508926, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07431785017251968 + }, + "val": { + "mean_centered_corr": -0.020286401733756065, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07910650968551636 + }, + "test": { + "mean_centered_corr": -0.02015332505106926, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07623301446437836 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 24, + 31, + 15, + 7, + 13, + 8, + 1, + 27, + 26, + 12, + 10, + 28, + 19, + 2, + 31, + 7, + 13, + 15, + 5, + 8, + 21, + 1, + 4, + 31 + ], + "head_selection_time": "0:00:00.885548" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_cf-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_cf-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_cf-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_cf-best-head-evaluation/0/report.json new file mode 100644 index 000000000..115bc741e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_cf-best-head-evaluation/0/report.json @@ -0,0 +1,633 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5175918577279733, + "mae": 0.40775081515312195, + "r2": 0.4978642463684082, + "score": -0.5175918577279733 + }, + "val": { + "rmse": 0.5572652691682626, + "mae": 0.44076380133628845, + "r2": 0.4143989086151123, + "score": -0.5572652691682626 + }, + "test": { + "rmse": 0.5524207799310179, + "mae": 0.4452172815799713, + "r2": 0.4313448667526245, + "score": -0.5524207799310179 + } + }, + "time": "0:15:04.586964", + "diversity": { + "train": { + "mean_centered_corr": -0.02016017585992813, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06105273962020874 + }, + "val": { + "mean_centered_corr": -0.01993357017636299, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06221357360482216 + }, + "test": { + "mean_centered_corr": -0.020221160724759102, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06099524348974228 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 11 + ], + "head_selection_time": "0:00:00.023192" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_cf-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_cf-evaluation/0.toml new file mode 100644 index 000000000..a8cfa8c75 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_cf-evaluation/0.toml @@ -0,0 +1,158 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.5 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "delivery-eta" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] +start_epoch = 16 +start_fraction = 0.5 +mode = "consensus_raw" + +[cf_fisd.member_groups] +xgb = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] +lgbm = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] +cat = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_cf-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_cf-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_cf-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_cf-evaluation/0/report.json new file mode 100644 index 000000000..aed8e342a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_cf-evaluation/0/report.json @@ -0,0 +1,629 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5150130136938161, + "mae": 0.40628108382225037, + "r2": 0.5028554797172546, + "score": -0.5150130136938161 + }, + "val": { + "rmse": 0.5559356776481553, + "mae": 0.441616415977478, + "r2": 0.4171900153160095, + "score": -0.5559356776481553 + }, + "test": { + "rmse": 0.55135486033622, + "mae": 0.445721298456192, + "r2": 0.43353718519210815, + "score": -0.55135486033622 + } + }, + "time": "0:15:04.586964", + "diversity": { + "train": { + "mean_centered_corr": -0.02016017585992813, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06105273962020874 + }, + "val": { + "mean_centered_corr": -0.01993357017636299, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06221357360482216 + }, + "test": { + "mean_centered_corr": -0.020221160724759102, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06099524348974228 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_cf-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_cf-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_cf-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_cf-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..e99c8f244 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_cf-greedy-heads-evaluation/0/report.json @@ -0,0 +1,641 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5148489632002486, + "mae": 0.40648993849754333, + "r2": 0.5031721591949463, + "score": -0.5148489632002486 + }, + "val": { + "rmse": 0.5553930389270275, + "mae": 0.4419495463371277, + "r2": 0.4183272123336792, + "score": -0.5553930389270275 + }, + "test": { + "rmse": 0.5519483119271499, + "mae": 0.4472074508666992, + "r2": 0.4323171377182007, + "score": -0.5519483119271499 + } + }, + "time": "0:15:04.586964", + "diversity": { + "train": { + "mean_centered_corr": -0.02016017585992813, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06105273962020874 + }, + "val": { + "mean_centered_corr": -0.01993357017636299, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06221357360482216 + }, + "test": { + "mean_centered_corr": -0.020221160724759102, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06099524348974228 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 11, + 10, + 26, + 21, + 15, + 16, + 31, + 19, + 13 + ], + "head_selection_time": "0:00:00.258661" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam-best-head-evaluation/0/report.json new file mode 100644 index 000000000..ad5047df6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam-best-head-evaluation/0/report.json @@ -0,0 +1,240 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam-evaluation/_esam_diagnostics/deliv_comb_rla_esam-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2730, + "metrics": { + "train": { + "rmse": 0.5099278042219015, + "mae": 0.4058413803577423, + "r2": 0.5126245617866516, + "score": -0.5099278042219015 + }, + "val": { + "rmse": 0.5591176650506209, + "mae": 0.44280362129211426, + "r2": 0.41049933433532715, + "score": -0.5591176650506209 + }, + "test": { + "rmse": 0.5583917212440949, + "mae": 0.4517841041088104, + "r2": 0.41898560523986816, + "score": -0.5583917212440949 + } + }, + "time": "0:30:12.293206", + "diversity": { + "train": { + "mean_centered_corr": -0.020068299025297165, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07235845923423767 + }, + "val": { + "mean_centered_corr": -0.019968286156654358, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0755666196346283 + }, + "test": { + "mean_centered_corr": -0.020015086978673935, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07387885451316833 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023391" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam-evaluation/0.toml new file mode 100644 index 000000000..178bf3917 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam-evaluation/0.toml @@ -0,0 +1,50 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.5 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.001 +esam_eps = 1e-12 +esam_adapter_only = false +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 8 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam-evaluation/0/report.json new file mode 100644 index 000000000..aa1fff617 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam-evaluation/0/report.json @@ -0,0 +1,236 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam-evaluation/_esam_diagnostics/deliv_comb_rla_esam-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2730, + "metrics": { + "train": { + "rmse": 0.5014604161558279, + "mae": 0.39543983340263367, + "r2": 0.5286759734153748, + "score": -0.5014604161558279 + }, + "val": { + "rmse": 0.5555664137600386, + "mae": 0.4397052228450775, + "r2": 0.41796398162841797, + "score": -0.5555664137600386 + }, + "test": { + "rmse": 0.5512279565776417, + "mae": 0.444046288728714, + "r2": 0.4337978959083557, + "score": -0.5512279565776417 + } + }, + "time": "0:30:12.293206", + "diversity": { + "train": { + "mean_centered_corr": -0.020068299025297165, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07235845923423767 + }, + "val": { + "mean_centered_corr": -0.019968286156654358, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0755666196346283 + }, + "test": { + "mean_centered_corr": -0.020015086978673935, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07387885451316833 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..2bc4c88e2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam-greedy-heads-evaluation/0/report.json @@ -0,0 +1,252 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam-evaluation/_esam_diagnostics/deliv_comb_rla_esam-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2730, + "metrics": { + "train": { + "rmse": 0.5013239141557576, + "mae": 0.3959278166294098, + "r2": 0.528932511806488, + "score": -0.5013239141557576 + }, + "val": { + "rmse": 0.5550917643214267, + "mae": 0.4404295086860657, + "r2": 0.41895806789398193, + "score": -0.5550917643214267 + }, + "test": { + "rmse": 0.5526470209655567, + "mae": 0.4466344714164734, + "r2": 0.4308789372444153, + "score": -0.5526470209655567 + } + }, + "time": "0:30:12.293206", + "diversity": { + "train": { + "mean_centered_corr": -0.020068299025297165, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07235845923423767 + }, + "val": { + "mean_centered_corr": -0.019968286156654358, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0755666196346283 + }, + "test": { + "mean_centered_corr": -0.020015086978673935, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07387885451316833 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13, + 20, + 21, + 11, + 8, + 12, + 7, + 23, + 19, + 15, + 31, + 28, + 22 + ], + "head_selection_time": "0:00:00.393915" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam_cf-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam_cf-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam_cf-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam_cf-best-head-evaluation/0/report.json new file mode 100644 index 000000000..7e536755e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam_cf-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam_cf-evaluation/_esam_diagnostics/deliv_comb_rla_esam_cf-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2730, + "metrics": { + "train": { + "rmse": 0.5099278042219015, + "mae": 0.4058413803577423, + "r2": 0.5126245617866516, + "score": -0.5099278042219015 + }, + "val": { + "rmse": 0.5591176650506209, + "mae": 0.44280362129211426, + "r2": 0.41049933433532715, + "score": -0.5591176650506209 + }, + "test": { + "rmse": 0.5583917212440949, + "mae": 0.4517841041088104, + "r2": 0.41898560523986816, + "score": -0.5583917212440949 + } + }, + "time": "0:31:32.729092", + "diversity": { + "train": { + "mean_centered_corr": -0.020068299025297165, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07235845923423767 + }, + "val": { + "mean_centered_corr": -0.019968286156654358, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0755666196346283 + }, + "test": { + "mean_centered_corr": -0.020015086978673935, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07387885451316833 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023934" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam_cf-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam_cf-evaluation/0.toml new file mode 100644 index 000000000..33ce5f9fe --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam_cf-evaluation/0.toml @@ -0,0 +1,168 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.5 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.001 +esam_eps = 1e-12 +esam_adapter_only = false +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 8 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "delivery-eta" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] +start_epoch = 16 +start_fraction = 0.5 +mode = "consensus_raw" + +[cf_fisd.member_groups] +xgb = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] +lgbm = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] +cat = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam_cf-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam_cf-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam_cf-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam_cf-evaluation/0/report.json new file mode 100644 index 000000000..5b29fb924 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam_cf-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam_cf-evaluation/_esam_diagnostics/deliv_comb_rla_esam_cf-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2730, + "metrics": { + "train": { + "rmse": 0.5014604161558279, + "mae": 0.39543983340263367, + "r2": 0.5286759734153748, + "score": -0.5014604161558279 + }, + "val": { + "rmse": 0.5555664137600386, + "mae": 0.4397052228450775, + "r2": 0.41796398162841797, + "score": -0.5555664137600386 + }, + "test": { + "rmse": 0.5512279565776417, + "mae": 0.444046288728714, + "r2": 0.4337978959083557, + "score": -0.5512279565776417 + } + }, + "time": "0:31:32.729092", + "diversity": { + "train": { + "mean_centered_corr": -0.020068299025297165, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07235845923423767 + }, + "val": { + "mean_centered_corr": -0.019968286156654358, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0755666196346283 + }, + "test": { + "mean_centered_corr": -0.020015086978673935, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07387885451316833 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam_cf-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam_cf-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam_cf-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam_cf-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..da59ef9ea --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam_cf-greedy-heads-evaluation/0/report.json @@ -0,0 +1,706 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam_cf-evaluation/_esam_diagnostics/deliv_comb_rla_esam_cf-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2730, + "metrics": { + "train": { + "rmse": 0.5013239141557576, + "mae": 0.3959278166294098, + "r2": 0.528932511806488, + "score": -0.5013239141557576 + }, + "val": { + "rmse": 0.5550917643214267, + "mae": 0.4404295086860657, + "r2": 0.41895806789398193, + "score": -0.5550917643214267 + }, + "test": { + "rmse": 0.5526470209655567, + "mae": 0.4466344714164734, + "r2": 0.4308789372444153, + "score": -0.5526470209655567 + } + }, + "time": "0:31:32.729092", + "diversity": { + "train": { + "mean_centered_corr": -0.020068299025297165, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07235845923423767 + }, + "val": { + "mean_centered_corr": -0.019968286156654358, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0755666196346283 + }, + "test": { + "mean_centered_corr": -0.020015086978673935, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07387885451316833 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13, + 20, + 21, + 11, + 8, + 12, + 7, + 23, + 19, + 15, + 31, + 28, + 22 + ], + "head_selection_time": "0:00:00.396932" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_consensus_raw-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_consensus_raw-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_consensus_raw-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_consensus_raw-best-head-evaluation/0/report.json new file mode 100644 index 000000000..011d702b0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_consensus_raw-best-head-evaluation/0/report.json @@ -0,0 +1,622 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5112224822345067, + "mae": 0.40589213371276855, + "r2": 0.5101465582847595, + "score": -0.5112224822345067 + }, + "val": { + "rmse": 0.5583881453279098, + "mae": 0.4439501166343689, + "r2": 0.412036657333374, + "score": -0.5583881453279098 + }, + "test": { + "rmse": 0.5557654209580059, + "mae": 0.4505227506160736, + "r2": 0.42443811893463135, + "score": -0.5557654209580059 + } + }, + "time": "0:07:50.667909", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023443" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_consensus_raw-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_consensus_raw-evaluation/0.toml new file mode 100644 index 000000000..e7d7fb4f3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_consensus_raw-evaluation/0.toml @@ -0,0 +1,151 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "delivery-eta" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] +start_epoch = 16 +start_fraction = 0.5 +mode = "consensus_raw" + +[cf_fisd.member_groups] +xgb = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] +lgbm = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] +cat = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_consensus_raw-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_consensus_raw-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_consensus_raw-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_consensus_raw-evaluation/0/report.json new file mode 100644 index 000000000..6337eebc5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_consensus_raw-evaluation/0/report.json @@ -0,0 +1,618 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5040057771555991, + "mae": 0.39488768577575684, + "r2": 0.5238790512084961, + "score": -0.5040057771555991 + }, + "val": { + "rmse": 0.5558749906446844, + "mae": 0.4398687481880188, + "r2": 0.4173172116279602, + "score": -0.5558749906446844 + }, + "test": { + "rmse": 0.5496162918403876, + "mae": 0.44109272956848145, + "r2": 0.4371039867401123, + "score": -0.5496162918403876 + } + }, + "time": "0:07:50.667909", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_consensus_raw-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_consensus_raw-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_consensus_raw-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_consensus_raw-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..9a27827e8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_consensus_raw-greedy-heads-evaluation/0/report.json @@ -0,0 +1,635 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5042286809765437, + "mae": 0.39605751633644104, + "r2": 0.5234577655792236, + "score": -0.5042286809765437 + }, + "val": { + "rmse": 0.5553348954063525, + "mae": 0.44016024470329285, + "r2": 0.4184489846229553, + "score": -0.5553348954063525 + }, + "test": { + "rmse": 0.550360417400049, + "mae": 0.44305530190467834, + "r2": 0.43557870388031006, + "score": -0.550360417400049 + } + }, + "time": "0:07:50.667909", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13, + 6, + 10, + 14, + 26, + 27, + 18, + 25, + 6, + 15, + 8, + 17, + 10, + 26 + ], + "head_selection_time": "0:00:00.439011" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_softmax-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_softmax-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_softmax-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_softmax-best-head-evaluation/0/report.json new file mode 100644 index 000000000..2869e0596 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_softmax-best-head-evaluation/0/report.json @@ -0,0 +1,454 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "softmax", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "softmax" + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "softmax", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "softmax" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5112224822345067, + "mae": 0.40589213371276855, + "r2": 0.5101465582847595, + "score": -0.5112224822345067 + }, + "val": { + "rmse": 0.5583881453279098, + "mae": 0.4439501166343689, + "r2": 0.412036657333374, + "score": -0.5583881453279098 + }, + "test": { + "rmse": 0.5557654209580059, + "mae": 0.4505227506160736, + "r2": 0.42443811893463135, + "score": -0.5557654209580059 + } + }, + "time": "0:07:32.728471", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023671" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_softmax-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_softmax-evaluation/0.toml new file mode 100644 index 000000000..92fc7d60c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_softmax-evaluation/0.toml @@ -0,0 +1,47 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 + +[cf_fisd] +lambda = 0.005 +variant = "softmax" +dataset_name = "delivery-eta" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] +start_epoch = 16 +start_fraction = 0.5 +mode = "softmax" diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_softmax-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_softmax-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_softmax-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_softmax-evaluation/0/report.json new file mode 100644 index 000000000..dc21703c3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_softmax-evaluation/0/report.json @@ -0,0 +1,450 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "softmax", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "softmax" + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "softmax", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "softmax" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5040057771555991, + "mae": 0.39488768577575684, + "r2": 0.5238790512084961, + "score": -0.5040057771555991 + }, + "val": { + "rmse": 0.5558749906446844, + "mae": 0.4398687481880188, + "r2": 0.4173172116279602, + "score": -0.5558749906446844 + }, + "test": { + "rmse": 0.5496162918403876, + "mae": 0.44109272956848145, + "r2": 0.4371039867401123, + "score": -0.5496162918403876 + } + }, + "time": "0:07:32.728471", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_softmax-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_softmax-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_softmax-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_softmax-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..1aac6f0ff --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_softmax-greedy-heads-evaluation/0/report.json @@ -0,0 +1,467 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "softmax", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "softmax" + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "softmax", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "softmax" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5042286809765437, + "mae": 0.39605751633644104, + "r2": 0.5234577655792236, + "score": -0.5042286809765437 + }, + "val": { + "rmse": 0.5553348954063525, + "mae": 0.44016024470329285, + "r2": 0.4184489846229553, + "score": -0.5553348954063525 + }, + "test": { + "rmse": 0.550360417400049, + "mae": 0.44305530190467834, + "r2": 0.43557870388031006, + "score": -0.550360417400049 + } + }, + "time": "0:07:32.728471", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13, + 6, + 10, + 14, + 26, + 27, + 18, + 25, + 6, + 15, + 8, + 17, + 10, + 26 + ], + "head_selection_time": "0:00:00.426444" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_consensus_raw-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_consensus_raw-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_consensus_raw-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_consensus_raw-best-head-evaluation/0/report.json new file mode 100644 index 000000000..4e75654ba --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_consensus_raw-best-head-evaluation/0/report.json @@ -0,0 +1,622 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5112224822345067, + "mae": 0.40589213371276855, + "r2": 0.5101465582847595, + "score": -0.5112224822345067 + }, + "val": { + "rmse": 0.5583881453279098, + "mae": 0.4439501166343689, + "r2": 0.412036657333374, + "score": -0.5583881453279098 + }, + "test": { + "rmse": 0.5557654209580059, + "mae": 0.4505227506160736, + "r2": 0.42443811893463135, + "score": -0.5557654209580059 + } + }, + "time": "0:07:47.591509", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023656" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_consensus_raw-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_consensus_raw-evaluation/0.toml new file mode 100644 index 000000000..853f0f9cb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_consensus_raw-evaluation/0.toml @@ -0,0 +1,151 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "delivery-eta" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] +start_epoch = 16 +start_fraction = 0.5 +mode = "consensus_raw" + +[cf_fisd.member_groups] +xgb = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] +lgbm = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] +cat = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_consensus_raw-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_consensus_raw-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_consensus_raw-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_consensus_raw-evaluation/0/report.json new file mode 100644 index 000000000..9e9a5a4c6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_consensus_raw-evaluation/0/report.json @@ -0,0 +1,618 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5040057771555991, + "mae": 0.39488768577575684, + "r2": 0.5238790512084961, + "score": -0.5040057771555991 + }, + "val": { + "rmse": 0.5558749906446844, + "mae": 0.4398687481880188, + "r2": 0.4173172116279602, + "score": -0.5558749906446844 + }, + "test": { + "rmse": 0.5496162918403876, + "mae": 0.44109272956848145, + "r2": 0.4371039867401123, + "score": -0.5496162918403876 + } + }, + "time": "0:07:47.591509", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_consensus_raw-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_consensus_raw-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_consensus_raw-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_consensus_raw-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..76d9893e9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_consensus_raw-greedy-heads-evaluation/0/report.json @@ -0,0 +1,635 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5042286809765437, + "mae": 0.39605751633644104, + "r2": 0.5234577655792236, + "score": -0.5042286809765437 + }, + "val": { + "rmse": 0.5553348954063525, + "mae": 0.44016024470329285, + "r2": 0.4184489846229553, + "score": -0.5553348954063525 + }, + "test": { + "rmse": 0.550360417400049, + "mae": 0.44305530190467834, + "r2": 0.43557870388031006, + "score": -0.550360417400049 + } + }, + "time": "0:07:47.591509", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13, + 6, + 10, + 14, + 26, + 27, + 18, + 25, + 6, + 15, + 8, + 17, + 10, + 26 + ], + "head_selection_time": "0:00:00.432736" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_softmax-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_softmax-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_softmax-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_softmax-best-head-evaluation/0/report.json new file mode 100644 index 000000000..b002af603 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_softmax-best-head-evaluation/0/report.json @@ -0,0 +1,454 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "softmax", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "softmax" + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "softmax", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "softmax" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5112224822345067, + "mae": 0.40589213371276855, + "r2": 0.5101465582847595, + "score": -0.5112224822345067 + }, + "val": { + "rmse": 0.5583881453279098, + "mae": 0.4439501166343689, + "r2": 0.412036657333374, + "score": -0.5583881453279098 + }, + "test": { + "rmse": 0.5557654209580059, + "mae": 0.4505227506160736, + "r2": 0.42443811893463135, + "score": -0.5557654209580059 + } + }, + "time": "0:07:33.670332", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023709" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_softmax-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_softmax-evaluation/0.toml new file mode 100644 index 000000000..89f0c2869 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_softmax-evaluation/0.toml @@ -0,0 +1,47 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 + +[cf_fisd] +lambda = 0.01 +variant = "softmax" +dataset_name = "delivery-eta" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] +start_epoch = 16 +start_fraction = 0.5 +mode = "softmax" diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_softmax-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_softmax-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_softmax-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_softmax-evaluation/0/report.json new file mode 100644 index 000000000..f18168d9c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_softmax-evaluation/0/report.json @@ -0,0 +1,450 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "softmax", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "softmax" + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "softmax", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "softmax" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5040057771555991, + "mae": 0.39488768577575684, + "r2": 0.5238790512084961, + "score": -0.5040057771555991 + }, + "val": { + "rmse": 0.5558749906446844, + "mae": 0.4398687481880188, + "r2": 0.4173172116279602, + "score": -0.5558749906446844 + }, + "test": { + "rmse": 0.5496162918403876, + "mae": 0.44109272956848145, + "r2": 0.4371039867401123, + "score": -0.5496162918403876 + } + }, + "time": "0:07:33.670332", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_softmax-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_softmax-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_softmax-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_softmax-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..fcc8f2184 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_softmax-greedy-heads-evaluation/0/report.json @@ -0,0 +1,467 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "softmax", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "softmax" + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "softmax", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "softmax" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5042286809765437, + "mae": 0.39605751633644104, + "r2": 0.5234577655792236, + "score": -0.5042286809765437 + }, + "val": { + "rmse": 0.5553348954063525, + "mae": 0.44016024470329285, + "r2": 0.4184489846229553, + "score": -0.5553348954063525 + }, + "test": { + "rmse": 0.550360417400049, + "mae": 0.44305530190467834, + "r2": 0.43557870388031006, + "score": -0.550360417400049 + } + }, + "time": "0:07:33.670332", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13, + 6, + 10, + 14, + 26, + 27, + 18, + 25, + 6, + 15, + 8, + 17, + 10, + 26 + ], + "head_selection_time": "0:00:00.432860" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_consensus_raw-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_consensus_raw-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_consensus_raw-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_consensus_raw-best-head-evaluation/0/report.json new file mode 100644 index 000000000..3a8df65db --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_consensus_raw-best-head-evaluation/0/report.json @@ -0,0 +1,622 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5112224822345067, + "mae": 0.40589213371276855, + "r2": 0.5101465582847595, + "score": -0.5112224822345067 + }, + "val": { + "rmse": 0.5583881453279098, + "mae": 0.4439501166343689, + "r2": 0.412036657333374, + "score": -0.5583881453279098 + }, + "test": { + "rmse": 0.5557654209580059, + "mae": 0.4505227506160736, + "r2": 0.42443811893463135, + "score": -0.5557654209580059 + } + }, + "time": "0:07:49.125262", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023170" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_consensus_raw-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_consensus_raw-evaluation/0.toml new file mode 100644 index 000000000..a1a0983ff --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_consensus_raw-evaluation/0.toml @@ -0,0 +1,151 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 + +[cf_fisd] +lambda = 0.02 +variant = "raw" +dataset_name = "delivery-eta" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] +start_epoch = 16 +start_fraction = 0.5 +mode = "consensus_raw" + +[cf_fisd.member_groups] +xgb = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] +lgbm = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] +cat = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_consensus_raw-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_consensus_raw-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_consensus_raw-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_consensus_raw-evaluation/0/report.json new file mode 100644 index 000000000..75a48e37f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_consensus_raw-evaluation/0/report.json @@ -0,0 +1,618 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5040057771555991, + "mae": 0.39488768577575684, + "r2": 0.5238790512084961, + "score": -0.5040057771555991 + }, + "val": { + "rmse": 0.5558749906446844, + "mae": 0.4398687481880188, + "r2": 0.4173172116279602, + "score": -0.5558749906446844 + }, + "test": { + "rmse": 0.5496162918403876, + "mae": 0.44109272956848145, + "r2": 0.4371039867401123, + "score": -0.5496162918403876 + } + }, + "time": "0:07:49.125262", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_consensus_raw-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_consensus_raw-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_consensus_raw-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_consensus_raw-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..554cf3c71 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_consensus_raw-greedy-heads-evaluation/0/report.json @@ -0,0 +1,635 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5042286809765437, + "mae": 0.39605751633644104, + "r2": 0.5234577655792236, + "score": -0.5042286809765437 + }, + "val": { + "rmse": 0.5553348954063525, + "mae": 0.44016024470329285, + "r2": 0.4184489846229553, + "score": -0.5553348954063525 + }, + "test": { + "rmse": 0.550360417400049, + "mae": 0.44305530190467834, + "r2": 0.43557870388031006, + "score": -0.550360417400049 + } + }, + "time": "0:07:49.125262", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13, + 6, + 10, + 14, + 26, + 27, + 18, + 25, + 6, + 15, + 8, + 17, + 10, + 26 + ], + "head_selection_time": "0:00:00.432808" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_softmax-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_softmax-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_softmax-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_softmax-best-head-evaluation/0/report.json new file mode 100644 index 000000000..8f7a20eb4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_softmax-best-head-evaluation/0/report.json @@ -0,0 +1,454 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "softmax", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "softmax" + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "softmax", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "softmax" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5112224822345067, + "mae": 0.40589213371276855, + "r2": 0.5101465582847595, + "score": -0.5112224822345067 + }, + "val": { + "rmse": 0.5583881453279098, + "mae": 0.4439501166343689, + "r2": 0.412036657333374, + "score": -0.5583881453279098 + }, + "test": { + "rmse": 0.5557654209580059, + "mae": 0.4505227506160736, + "r2": 0.42443811893463135, + "score": -0.5557654209580059 + } + }, + "time": "0:07:31.619857", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023385" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_softmax-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_softmax-evaluation/0.toml new file mode 100644 index 000000000..2005313ed --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_softmax-evaluation/0.toml @@ -0,0 +1,47 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 + +[cf_fisd] +lambda = 0.02 +variant = "softmax" +dataset_name = "delivery-eta" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] +start_epoch = 16 +start_fraction = 0.5 +mode = "softmax" diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_softmax-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_softmax-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_softmax-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_softmax-evaluation/0/report.json new file mode 100644 index 000000000..ba314e909 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_softmax-evaluation/0/report.json @@ -0,0 +1,450 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "softmax", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "softmax" + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "softmax", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "softmax" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5040057771555991, + "mae": 0.39488768577575684, + "r2": 0.5238790512084961, + "score": -0.5040057771555991 + }, + "val": { + "rmse": 0.5558749906446844, + "mae": 0.4398687481880188, + "r2": 0.4173172116279602, + "score": -0.5558749906446844 + }, + "test": { + "rmse": 0.5496162918403876, + "mae": 0.44109272956848145, + "r2": 0.4371039867401123, + "score": -0.5496162918403876 + } + }, + "time": "0:07:31.619857", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_softmax-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_softmax-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_softmax-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_softmax-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..3b5fe62a8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_softmax-greedy-heads-evaluation/0/report.json @@ -0,0 +1,467 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "softmax", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "softmax" + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "softmax", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "softmax" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5042286809765437, + "mae": 0.39605751633644104, + "r2": 0.5234577655792236, + "score": -0.5042286809765437 + }, + "val": { + "rmse": 0.5553348954063525, + "mae": 0.44016024470329285, + "r2": 0.4184489846229553, + "score": -0.5553348954063525 + }, + "test": { + "rmse": 0.550360417400049, + "mae": 0.44305530190467834, + "r2": 0.43557870388031006, + "score": -0.550360417400049 + } + }, + "time": "0:07:31.619857", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13, + 6, + 10, + 14, + 26, + 27, + 18, + 25, + 6, + 15, + 8, + 17, + 10, + 26 + ], + "head_selection_time": "0:00:00.443035" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adapterf-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adapterf-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adapterf-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adapterf-best-head-evaluation/0/report.json new file mode 100644 index 000000000..36228be09 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adapterf-best-head-evaluation/0/report.json @@ -0,0 +1,229 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adapterf-evaluation/_esam_diagnostics/deliv_esam_rho0p00025_sf0p25_adapterf-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5112224822345067, + "mae": 0.40589213371276855, + "r2": 0.5101465582847595, + "score": -0.5112224822345067 + }, + "val": { + "rmse": 0.5583881453279098, + "mae": 0.4439501166343689, + "r2": 0.412036657333374, + "score": -0.5583881453279098 + }, + "test": { + "rmse": 0.5557654209580059, + "mae": 0.4505227506160736, + "r2": 0.42443811893463135, + "score": -0.5557654209580059 + } + }, + "time": "0:11:54.514683", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023259" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adapterf-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adapterf-evaluation/0.toml new file mode 100644 index 000000000..ce06a82a3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adapterf-evaluation/0.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.00025 +esam_eps = 1e-12 +esam_adapter_only = false +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 8 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adapterf-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adapterf-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adapterf-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adapterf-evaluation/0/report.json new file mode 100644 index 000000000..4178bd75e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adapterf-evaluation/0/report.json @@ -0,0 +1,225 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adapterf-evaluation/_esam_diagnostics/deliv_esam_rho0p00025_sf0p25_adapterf-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5040057771555991, + "mae": 0.39488768577575684, + "r2": 0.5238790512084961, + "score": -0.5040057771555991 + }, + "val": { + "rmse": 0.5558749906446844, + "mae": 0.4398687481880188, + "r2": 0.4173172116279602, + "score": -0.5558749906446844 + }, + "test": { + "rmse": 0.5496162918403876, + "mae": 0.44109272956848145, + "r2": 0.4371039867401123, + "score": -0.5496162918403876 + } + }, + "time": "0:11:54.514683", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adapterf-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adapterf-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adapterf-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adapterf-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..7daaef5ec --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adapterf-greedy-heads-evaluation/0/report.json @@ -0,0 +1,242 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adapterf-evaluation/_esam_diagnostics/deliv_esam_rho0p00025_sf0p25_adapterf-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5042286809765437, + "mae": 0.39605751633644104, + "r2": 0.5234577655792236, + "score": -0.5042286809765437 + }, + "val": { + "rmse": 0.5553348954063525, + "mae": 0.44016024470329285, + "r2": 0.4184489846229553, + "score": -0.5553348954063525 + }, + "test": { + "rmse": 0.550360417400049, + "mae": 0.44305530190467834, + "r2": 0.43557870388031006, + "score": -0.550360417400049 + } + }, + "time": "0:11:54.514683", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13, + 6, + 10, + 14, + 26, + 27, + 18, + 25, + 6, + 15, + 8, + 17, + 10, + 26 + ], + "head_selection_time": "0:00:00.437296" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adaptert-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adaptert-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adaptert-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adaptert-best-head-evaluation/0/report.json new file mode 100644 index 000000000..21e979ca4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adaptert-best-head-evaluation/0/report.json @@ -0,0 +1,178 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adaptert-evaluation/_esam_diagnostics/deliv_esam_rho0p00025_sf0p25_adaptert-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5112224530864115, + "mae": 0.40589213371276855, + "r2": 0.5101466178894043, + "score": -0.5112224530864115 + }, + "val": { + "rmse": 0.5583881453279098, + "mae": 0.4439501166343689, + "r2": 0.412036657333374, + "score": -0.5583881453279098 + }, + "test": { + "rmse": 0.5557654209580059, + "mae": 0.4505227506160736, + "r2": 0.42443811893463135, + "score": -0.5557654209580059 + } + }, + "time": "0:10:58.902882", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442888140678406, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958992034196854 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023427" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adaptert-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adaptert-evaluation/0.toml new file mode 100644 index 000000000..e552d66e3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adaptert-evaluation/0.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.00025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 8 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adaptert-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adaptert-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adaptert-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adaptert-evaluation/0/report.json new file mode 100644 index 000000000..cfccf217f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adaptert-evaluation/0/report.json @@ -0,0 +1,174 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adaptert-evaluation/_esam_diagnostics/deliv_esam_rho0p00025_sf0p25_adaptert-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5040057771555991, + "mae": 0.39488768577575684, + "r2": 0.5238790512084961, + "score": -0.5040057771555991 + }, + "val": { + "rmse": 0.5558750174513605, + "mae": 0.4398687481880188, + "r2": 0.41731715202331543, + "score": -0.5558750174513605 + }, + "test": { + "rmse": 0.5496162918403876, + "mae": 0.44109272956848145, + "r2": 0.4371039867401123, + "score": -0.5496162918403876 + } + }, + "time": "0:10:58.902882", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442888140678406, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958992034196854 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adaptert-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adaptert-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adaptert-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adaptert-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..07089b738 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adaptert-greedy-heads-evaluation/0/report.json @@ -0,0 +1,191 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adaptert-evaluation/_esam_diagnostics/deliv_esam_rho0p00025_sf0p25_adaptert-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.50422871052893, + "mae": 0.39605751633644104, + "r2": 0.5234577655792236, + "score": -0.50422871052893 + }, + "val": { + "rmse": 0.5553348954063525, + "mae": 0.44016024470329285, + "r2": 0.4184489846229553, + "score": -0.5553348954063525 + }, + "test": { + "rmse": 0.550360417400049, + "mae": 0.44305530190467834, + "r2": 0.43557870388031006, + "score": -0.550360417400049 + } + }, + "time": "0:10:58.902882", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442888140678406, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958992034196854 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13, + 6, + 10, + 14, + 26, + 27, + 18, + 25, + 6, + 15, + 8, + 17, + 10, + 26 + ], + "head_selection_time": "0:00:00.433917" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adapterf-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adapterf-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adapterf-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adapterf-best-head-evaluation/0/report.json new file mode 100644 index 000000000..1e121ca71 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adapterf-best-head-evaluation/0/report.json @@ -0,0 +1,229 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adapterf-evaluation/_esam_diagnostics/deliv_esam_rho0p00025_sf0p5_adapterf-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5112224822345067, + "mae": 0.40589213371276855, + "r2": 0.5101465582847595, + "score": -0.5112224822345067 + }, + "val": { + "rmse": 0.5583881453279098, + "mae": 0.4439501166343689, + "r2": 0.412036657333374, + "score": -0.5583881453279098 + }, + "test": { + "rmse": 0.5557654209580059, + "mae": 0.4505227506160736, + "r2": 0.42443811893463135, + "score": -0.5557654209580059 + } + }, + "time": "0:09:36.298262", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023756" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adapterf-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adapterf-evaluation/0.toml new file mode 100644 index 000000000..81af33fff --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adapterf-evaluation/0.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.00025 +esam_eps = 1e-12 +esam_adapter_only = false +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 16 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adapterf-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adapterf-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adapterf-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adapterf-evaluation/0/report.json new file mode 100644 index 000000000..b05705786 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adapterf-evaluation/0/report.json @@ -0,0 +1,225 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adapterf-evaluation/_esam_diagnostics/deliv_esam_rho0p00025_sf0p5_adapterf-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5040057771555991, + "mae": 0.39488768577575684, + "r2": 0.5238790512084961, + "score": -0.5040057771555991 + }, + "val": { + "rmse": 0.5558749906446844, + "mae": 0.4398687481880188, + "r2": 0.4173172116279602, + "score": -0.5558749906446844 + }, + "test": { + "rmse": 0.5496162918403876, + "mae": 0.44109272956848145, + "r2": 0.4371039867401123, + "score": -0.5496162918403876 + } + }, + "time": "0:09:36.298262", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adapterf-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adapterf-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adapterf-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adapterf-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..d1f58753b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adapterf-greedy-heads-evaluation/0/report.json @@ -0,0 +1,242 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adapterf-evaluation/_esam_diagnostics/deliv_esam_rho0p00025_sf0p5_adapterf-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5042286809765437, + "mae": 0.39605751633644104, + "r2": 0.5234577655792236, + "score": -0.5042286809765437 + }, + "val": { + "rmse": 0.5553348954063525, + "mae": 0.44016024470329285, + "r2": 0.4184489846229553, + "score": -0.5553348954063525 + }, + "test": { + "rmse": 0.550360417400049, + "mae": 0.44305530190467834, + "r2": 0.43557870388031006, + "score": -0.550360417400049 + } + }, + "time": "0:09:36.298262", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13, + 6, + 10, + 14, + 26, + 27, + 18, + 25, + 6, + 15, + 8, + 17, + 10, + 26 + ], + "head_selection_time": "0:00:00.434693" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adaptert-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adaptert-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adaptert-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adaptert-best-head-evaluation/0/report.json new file mode 100644 index 000000000..b3701dc27 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adaptert-best-head-evaluation/0/report.json @@ -0,0 +1,178 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adaptert-evaluation/_esam_diagnostics/deliv_esam_rho0p00025_sf0p5_adaptert-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5112224530864115, + "mae": 0.40589213371276855, + "r2": 0.5101466178894043, + "score": -0.5112224530864115 + }, + "val": { + "rmse": 0.5583881453279098, + "mae": 0.4439501166343689, + "r2": 0.412036657333374, + "score": -0.5583881453279098 + }, + "test": { + "rmse": 0.5557654209580059, + "mae": 0.4505227506160736, + "r2": 0.42443811893463135, + "score": -0.5557654209580059 + } + }, + "time": "0:09:06.499354", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442888140678406, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958992034196854 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023446" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adaptert-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adaptert-evaluation/0.toml new file mode 100644 index 000000000..66221114c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adaptert-evaluation/0.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.00025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 16 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adaptert-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adaptert-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adaptert-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adaptert-evaluation/0/report.json new file mode 100644 index 000000000..7e408ea6d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adaptert-evaluation/0/report.json @@ -0,0 +1,174 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adaptert-evaluation/_esam_diagnostics/deliv_esam_rho0p00025_sf0p5_adaptert-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5040057771555991, + "mae": 0.39488768577575684, + "r2": 0.5238790512084961, + "score": -0.5040057771555991 + }, + "val": { + "rmse": 0.5558750174513605, + "mae": 0.4398687481880188, + "r2": 0.41731715202331543, + "score": -0.5558750174513605 + }, + "test": { + "rmse": 0.5496162918403876, + "mae": 0.44109272956848145, + "r2": 0.4371039867401123, + "score": -0.5496162918403876 + } + }, + "time": "0:09:06.499354", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442888140678406, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958992034196854 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adaptert-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adaptert-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adaptert-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adaptert-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..7de1edf8d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adaptert-greedy-heads-evaluation/0/report.json @@ -0,0 +1,191 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adaptert-evaluation/_esam_diagnostics/deliv_esam_rho0p00025_sf0p5_adaptert-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.50422871052893, + "mae": 0.39605751633644104, + "r2": 0.5234577655792236, + "score": -0.50422871052893 + }, + "val": { + "rmse": 0.5553348954063525, + "mae": 0.44016024470329285, + "r2": 0.4184489846229553, + "score": -0.5553348954063525 + }, + "test": { + "rmse": 0.550360417400049, + "mae": 0.44305530190467834, + "r2": 0.43557870388031006, + "score": -0.550360417400049 + } + }, + "time": "0:09:06.499354", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442888140678406, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958992034196854 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13, + 6, + 10, + 14, + 26, + 27, + 18, + 25, + 6, + 15, + 8, + 17, + 10, + 26 + ], + "head_selection_time": "0:00:00.437358" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adapterf-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adapterf-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adapterf-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adapterf-best-head-evaluation/0/report.json new file mode 100644 index 000000000..06abf0f83 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adapterf-best-head-evaluation/0/report.json @@ -0,0 +1,229 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adapterf-evaluation/_esam_diagnostics/deliv_esam_rho0p0005_sf0p25_adapterf-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5112224822345067, + "mae": 0.40589213371276855, + "r2": 0.5101465582847595, + "score": -0.5112224822345067 + }, + "val": { + "rmse": 0.5583881453279098, + "mae": 0.4439501166343689, + "r2": 0.412036657333374, + "score": -0.5583881453279098 + }, + "test": { + "rmse": 0.5557654209580059, + "mae": 0.4505227506160736, + "r2": 0.42443811893463135, + "score": -0.5557654209580059 + } + }, + "time": "0:11:58.210461", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023576" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adapterf-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adapterf-evaluation/0.toml new file mode 100644 index 000000000..f6ebc3b63 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adapterf-evaluation/0.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.0005 +esam_eps = 1e-12 +esam_adapter_only = false +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 8 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adapterf-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adapterf-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adapterf-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adapterf-evaluation/0/report.json new file mode 100644 index 000000000..82de312dc --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adapterf-evaluation/0/report.json @@ -0,0 +1,225 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adapterf-evaluation/_esam_diagnostics/deliv_esam_rho0p0005_sf0p25_adapterf-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5040057771555991, + "mae": 0.39488768577575684, + "r2": 0.5238790512084961, + "score": -0.5040057771555991 + }, + "val": { + "rmse": 0.5558749906446844, + "mae": 0.4398687481880188, + "r2": 0.4173172116279602, + "score": -0.5558749906446844 + }, + "test": { + "rmse": 0.5496162918403876, + "mae": 0.44109272956848145, + "r2": 0.4371039867401123, + "score": -0.5496162918403876 + } + }, + "time": "0:11:58.210461", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adapterf-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adapterf-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adapterf-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adapterf-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..dcac0964e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adapterf-greedy-heads-evaluation/0/report.json @@ -0,0 +1,242 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adapterf-evaluation/_esam_diagnostics/deliv_esam_rho0p0005_sf0p25_adapterf-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5042286809765437, + "mae": 0.39605751633644104, + "r2": 0.5234577655792236, + "score": -0.5042286809765437 + }, + "val": { + "rmse": 0.5553348954063525, + "mae": 0.44016024470329285, + "r2": 0.4184489846229553, + "score": -0.5553348954063525 + }, + "test": { + "rmse": 0.550360417400049, + "mae": 0.44305530190467834, + "r2": 0.43557870388031006, + "score": -0.550360417400049 + } + }, + "time": "0:11:58.210461", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13, + 6, + 10, + 14, + 26, + 27, + 18, + 25, + 6, + 15, + 8, + 17, + 10, + 26 + ], + "head_selection_time": "0:00:00.429491" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adaptert-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adaptert-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adaptert-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adaptert-best-head-evaluation/0/report.json new file mode 100644 index 000000000..596adaf42 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adaptert-best-head-evaluation/0/report.json @@ -0,0 +1,178 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adaptert-evaluation/_esam_diagnostics/deliv_esam_rho0p0005_sf0p25_adaptert-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5112224530864115, + "mae": 0.40589213371276855, + "r2": 0.5101466178894043, + "score": -0.5112224530864115 + }, + "val": { + "rmse": 0.5583881453279098, + "mae": 0.4439501166343689, + "r2": 0.412036657333374, + "score": -0.5583881453279098 + }, + "test": { + "rmse": 0.5557654209580059, + "mae": 0.4505227506160736, + "r2": 0.42443811893463135, + "score": -0.5557654209580059 + } + }, + "time": "0:11:00.793096", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442888140678406, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958992034196854 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023362" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adaptert-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adaptert-evaluation/0.toml new file mode 100644 index 000000000..067d1c90d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adaptert-evaluation/0.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.0005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 8 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adaptert-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adaptert-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adaptert-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adaptert-evaluation/0/report.json new file mode 100644 index 000000000..953f014db --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adaptert-evaluation/0/report.json @@ -0,0 +1,174 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adaptert-evaluation/_esam_diagnostics/deliv_esam_rho0p0005_sf0p25_adaptert-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5040057771555991, + "mae": 0.39488768577575684, + "r2": 0.5238790512084961, + "score": -0.5040057771555991 + }, + "val": { + "rmse": 0.5558750174513605, + "mae": 0.4398687481880188, + "r2": 0.41731715202331543, + "score": -0.5558750174513605 + }, + "test": { + "rmse": 0.5496162918403876, + "mae": 0.44109272956848145, + "r2": 0.4371039867401123, + "score": -0.5496162918403876 + } + }, + "time": "0:11:00.793096", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442888140678406, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958992034196854 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adaptert-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adaptert-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adaptert-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adaptert-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..6acf4b8e9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adaptert-greedy-heads-evaluation/0/report.json @@ -0,0 +1,191 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adaptert-evaluation/_esam_diagnostics/deliv_esam_rho0p0005_sf0p25_adaptert-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.50422871052893, + "mae": 0.39605751633644104, + "r2": 0.5234577655792236, + "score": -0.50422871052893 + }, + "val": { + "rmse": 0.5553348954063525, + "mae": 0.44016024470329285, + "r2": 0.4184489846229553, + "score": -0.5553348954063525 + }, + "test": { + "rmse": 0.550360417400049, + "mae": 0.44305530190467834, + "r2": 0.43557870388031006, + "score": -0.550360417400049 + } + }, + "time": "0:11:00.793096", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442888140678406, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958992034196854 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13, + 6, + 10, + 14, + 26, + 27, + 18, + 25, + 6, + 15, + 8, + 17, + 10, + 26 + ], + "head_selection_time": "0:00:00.431835" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adapterf-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adapterf-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adapterf-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adapterf-best-head-evaluation/0/report.json new file mode 100644 index 000000000..3fdfbbd30 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adapterf-best-head-evaluation/0/report.json @@ -0,0 +1,229 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adapterf-evaluation/_esam_diagnostics/deliv_esam_rho0p0005_sf0p5_adapterf-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5112224822345067, + "mae": 0.40589213371276855, + "r2": 0.5101465582847595, + "score": -0.5112224822345067 + }, + "val": { + "rmse": 0.5583881453279098, + "mae": 0.4439501166343689, + "r2": 0.412036657333374, + "score": -0.5583881453279098 + }, + "test": { + "rmse": 0.5557654209580059, + "mae": 0.4505227506160736, + "r2": 0.42443811893463135, + "score": -0.5557654209580059 + } + }, + "time": "0:09:36.252294", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023761" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adapterf-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adapterf-evaluation/0.toml new file mode 100644 index 000000000..6ac8d4057 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adapterf-evaluation/0.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.0005 +esam_eps = 1e-12 +esam_adapter_only = false +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 16 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adapterf-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adapterf-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adapterf-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adapterf-evaluation/0/report.json new file mode 100644 index 000000000..be11ffae9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adapterf-evaluation/0/report.json @@ -0,0 +1,225 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adapterf-evaluation/_esam_diagnostics/deliv_esam_rho0p0005_sf0p5_adapterf-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5040057771555991, + "mae": 0.39488768577575684, + "r2": 0.5238790512084961, + "score": -0.5040057771555991 + }, + "val": { + "rmse": 0.5558749906446844, + "mae": 0.4398687481880188, + "r2": 0.4173172116279602, + "score": -0.5558749906446844 + }, + "test": { + "rmse": 0.5496162918403876, + "mae": 0.44109272956848145, + "r2": 0.4371039867401123, + "score": -0.5496162918403876 + } + }, + "time": "0:09:36.252294", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adapterf-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adapterf-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adapterf-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adapterf-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..0a18f0c54 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adapterf-greedy-heads-evaluation/0/report.json @@ -0,0 +1,242 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adapterf-evaluation/_esam_diagnostics/deliv_esam_rho0p0005_sf0p5_adapterf-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5042286809765437, + "mae": 0.39605751633644104, + "r2": 0.5234577655792236, + "score": -0.5042286809765437 + }, + "val": { + "rmse": 0.5553348954063525, + "mae": 0.44016024470329285, + "r2": 0.4184489846229553, + "score": -0.5553348954063525 + }, + "test": { + "rmse": 0.550360417400049, + "mae": 0.44305530190467834, + "r2": 0.43557870388031006, + "score": -0.550360417400049 + } + }, + "time": "0:09:36.252294", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13, + 6, + 10, + 14, + 26, + 27, + 18, + 25, + 6, + 15, + 8, + 17, + 10, + 26 + ], + "head_selection_time": "0:00:00.429945" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adaptert-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adaptert-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adaptert-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adaptert-best-head-evaluation/0/report.json new file mode 100644 index 000000000..45c3e689f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adaptert-best-head-evaluation/0/report.json @@ -0,0 +1,178 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adaptert-evaluation/_esam_diagnostics/deliv_esam_rho0p0005_sf0p5_adaptert-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5112224530864115, + "mae": 0.40589213371276855, + "r2": 0.5101466178894043, + "score": -0.5112224530864115 + }, + "val": { + "rmse": 0.5583881453279098, + "mae": 0.4439501166343689, + "r2": 0.412036657333374, + "score": -0.5583881453279098 + }, + "test": { + "rmse": 0.5557654209580059, + "mae": 0.4505227506160736, + "r2": 0.42443811893463135, + "score": -0.5557654209580059 + } + }, + "time": "0:09:09.025241", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442888140678406, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958992034196854 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023020" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adaptert-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adaptert-evaluation/0.toml new file mode 100644 index 000000000..867546608 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adaptert-evaluation/0.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.0005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 16 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adaptert-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adaptert-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adaptert-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adaptert-evaluation/0/report.json new file mode 100644 index 000000000..9aea75d97 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adaptert-evaluation/0/report.json @@ -0,0 +1,174 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adaptert-evaluation/_esam_diagnostics/deliv_esam_rho0p0005_sf0p5_adaptert-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5040057771555991, + "mae": 0.39488768577575684, + "r2": 0.5238790512084961, + "score": -0.5040057771555991 + }, + "val": { + "rmse": 0.5558750174513605, + "mae": 0.4398687481880188, + "r2": 0.41731715202331543, + "score": -0.5558750174513605 + }, + "test": { + "rmse": 0.5496162918403876, + "mae": 0.44109272956848145, + "r2": 0.4371039867401123, + "score": -0.5496162918403876 + } + }, + "time": "0:09:09.025241", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442888140678406, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958992034196854 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adaptert-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adaptert-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adaptert-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adaptert-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..690c5e88a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adaptert-greedy-heads-evaluation/0/report.json @@ -0,0 +1,191 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adaptert-evaluation/_esam_diagnostics/deliv_esam_rho0p0005_sf0p5_adaptert-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.50422871052893, + "mae": 0.39605751633644104, + "r2": 0.5234577655792236, + "score": -0.50422871052893 + }, + "val": { + "rmse": 0.5553348954063525, + "mae": 0.44016024470329285, + "r2": 0.4184489846229553, + "score": -0.5553348954063525 + }, + "test": { + "rmse": 0.550360417400049, + "mae": 0.44305530190467834, + "r2": 0.43557870388031006, + "score": -0.550360417400049 + } + }, + "time": "0:09:09.025241", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442888140678406, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958992034196854 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13, + 6, + 10, + 14, + 26, + 27, + 18, + 25, + 6, + 15, + 8, + 17, + 10, + 26 + ], + "head_selection_time": "0:00:00.422502" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adapterf-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adapterf-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adapterf-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adapterf-best-head-evaluation/0/report.json new file mode 100644 index 000000000..3e99b717d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adapterf-best-head-evaluation/0/report.json @@ -0,0 +1,229 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adapterf-evaluation/_esam_diagnostics/deliv_esam_rho0p001_sf0p25_adapterf-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2730, + "metrics": { + "train": { + "rmse": 0.5056157353876229, + "mae": 0.3998609483242035, + "r2": 0.5208324193954468, + "score": -0.5056157353876229 + }, + "val": { + "rmse": 0.5596782586864594, + "mae": 0.44516822695732117, + "r2": 0.4093165993690491, + "score": -0.5596782586864594 + }, + "test": { + "rmse": 0.5565908577070521, + "mae": 0.45004433393478394, + "r2": 0.42272716760635376, + "score": -0.5565908577070521 + } + }, + "time": "0:13:05.256760", + "diversity": { + "train": { + "mean_centered_corr": -0.020204994827508926, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07431785017251968 + }, + "val": { + "mean_centered_corr": -0.020286401733756065, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07910650968551636 + }, + "test": { + "mean_centered_corr": -0.02015332505106926, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07623301446437836 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 24 + ], + "head_selection_time": "0:00:00.023424" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adapterf-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adapterf-evaluation/0.toml new file mode 100644 index 000000000..474196b1e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adapterf-evaluation/0.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.001 +esam_eps = 1e-12 +esam_adapter_only = false +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 8 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adapterf-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adapterf-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adapterf-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adapterf-evaluation/0/report.json new file mode 100644 index 000000000..923d84058 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adapterf-evaluation/0/report.json @@ -0,0 +1,225 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adapterf-evaluation/_esam_diagnostics/deliv_esam_rho0p001_sf0p25_adapterf-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2730, + "metrics": { + "train": { + "rmse": 0.4975037099949126, + "mae": 0.39285650849342346, + "r2": 0.5360844135284424, + "score": -0.4975037099949126 + }, + "val": { + "rmse": 0.5558269510040955, + "mae": 0.44211816787719727, + "r2": 0.4174179434776306, + "score": -0.5558269510040955 + }, + "test": { + "rmse": 0.5527269612276788, + "mae": 0.44693246483802795, + "r2": 0.4307142496109009, + "score": -0.5527269612276788 + } + }, + "time": "0:13:05.256760", + "diversity": { + "train": { + "mean_centered_corr": -0.020204994827508926, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07431785017251968 + }, + "val": { + "mean_centered_corr": -0.020286401733756065, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07910650968551636 + }, + "test": { + "mean_centered_corr": -0.02015332505106926, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07623301446437836 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adapterf-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adapterf-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adapterf-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adapterf-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..f8a336e9b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adapterf-greedy-heads-evaluation/0/report.json @@ -0,0 +1,252 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adapterf-evaluation/_esam_diagnostics/deliv_esam_rho0p001_sf0p25_adapterf-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2730, + "metrics": { + "train": { + "rmse": 0.4981544378477528, + "mae": 0.3936895430088043, + "r2": 0.5348700881004333, + "score": -0.4981544378477528 + }, + "val": { + "rmse": 0.555452544543325, + "mae": 0.44208067655563354, + "r2": 0.4182025194168091, + "score": -0.555452544543325 + }, + "test": { + "rmse": 0.5532771704027001, + "mae": 0.44795727729797363, + "r2": 0.42958033084869385, + "score": -0.5532771704027001 + } + }, + "time": "0:13:05.256760", + "diversity": { + "train": { + "mean_centered_corr": -0.020204994827508926, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07431785017251968 + }, + "val": { + "mean_centered_corr": -0.020286401733756065, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07910650968551636 + }, + "test": { + "mean_centered_corr": -0.02015332505106926, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07623301446437836 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 24, + 31, + 15, + 7, + 13, + 8, + 1, + 27, + 26, + 12, + 10, + 28, + 19, + 2, + 31, + 7, + 13, + 15, + 5, + 8, + 21, + 1, + 4, + 31 + ], + "head_selection_time": "0:00:00.922852" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adaptert-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adaptert-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adaptert-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adaptert-best-head-evaluation/0/report.json new file mode 100644 index 000000000..ced6ae76a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adaptert-best-head-evaluation/0/report.json @@ -0,0 +1,178 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adaptert-evaluation/_esam_diagnostics/deliv_esam_rho0p001_sf0p25_adaptert-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5112224530864115, + "mae": 0.40589213371276855, + "r2": 0.5101466178894043, + "score": -0.5112224530864115 + }, + "val": { + "rmse": 0.5583881453279098, + "mae": 0.4439501166343689, + "r2": 0.412036657333374, + "score": -0.5583881453279098 + }, + "test": { + "rmse": 0.5557654209580059, + "mae": 0.4505227506160736, + "r2": 0.42443811893463135, + "score": -0.5557654209580059 + } + }, + "time": "0:10:59.649752", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442888140678406, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958992034196854 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023376" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adaptert-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adaptert-evaluation/0.toml new file mode 100644 index 000000000..d3c1f452d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adaptert-evaluation/0.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.001 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 8 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adaptert-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adaptert-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adaptert-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adaptert-evaluation/0/report.json new file mode 100644 index 000000000..835158685 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adaptert-evaluation/0/report.json @@ -0,0 +1,174 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adaptert-evaluation/_esam_diagnostics/deliv_esam_rho0p001_sf0p25_adaptert-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5040057771555991, + "mae": 0.39488768577575684, + "r2": 0.5238790512084961, + "score": -0.5040057771555991 + }, + "val": { + "rmse": 0.5558750174513605, + "mae": 0.4398687481880188, + "r2": 0.41731715202331543, + "score": -0.5558750174513605 + }, + "test": { + "rmse": 0.5496162918403876, + "mae": 0.44109272956848145, + "r2": 0.4371039867401123, + "score": -0.5496162918403876 + } + }, + "time": "0:10:59.649752", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442888140678406, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958992034196854 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adaptert-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adaptert-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adaptert-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adaptert-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..998291c51 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adaptert-greedy-heads-evaluation/0/report.json @@ -0,0 +1,191 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adaptert-evaluation/_esam_diagnostics/deliv_esam_rho0p001_sf0p25_adaptert-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.50422871052893, + "mae": 0.39605751633644104, + "r2": 0.5234577655792236, + "score": -0.50422871052893 + }, + "val": { + "rmse": 0.5553348954063525, + "mae": 0.44016024470329285, + "r2": 0.4184489846229553, + "score": -0.5553348954063525 + }, + "test": { + "rmse": 0.550360417400049, + "mae": 0.44305530190467834, + "r2": 0.43557870388031006, + "score": -0.550360417400049 + } + }, + "time": "0:10:59.649752", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442888140678406, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958992034196854 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13, + 6, + 10, + 14, + 26, + 27, + 18, + 25, + 6, + 15, + 8, + 17, + 10, + 26 + ], + "head_selection_time": "0:00:00.425940" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adapterf-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adapterf-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adapterf-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adapterf-best-head-evaluation/0/report.json new file mode 100644 index 000000000..18e09303d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adapterf-best-head-evaluation/0/report.json @@ -0,0 +1,229 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adapterf-evaluation/_esam_diagnostics/deliv_esam_rho0p001_sf0p5_adapterf-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5112224822345067, + "mae": 0.40589213371276855, + "r2": 0.5101465582847595, + "score": -0.5112224822345067 + }, + "val": { + "rmse": 0.5583881453279098, + "mae": 0.4439501166343689, + "r2": 0.412036657333374, + "score": -0.5583881453279098 + }, + "test": { + "rmse": 0.5557654209580059, + "mae": 0.4505227506160736, + "r2": 0.42443811893463135, + "score": -0.5557654209580059 + } + }, + "time": "0:09:34.700424", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023611" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adapterf-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adapterf-evaluation/0.toml new file mode 100644 index 000000000..a5511d194 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adapterf-evaluation/0.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.001 +esam_eps = 1e-12 +esam_adapter_only = false +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 16 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adapterf-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adapterf-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adapterf-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adapterf-evaluation/0/report.json new file mode 100644 index 000000000..1bc764eaa --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adapterf-evaluation/0/report.json @@ -0,0 +1,225 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adapterf-evaluation/_esam_diagnostics/deliv_esam_rho0p001_sf0p5_adapterf-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5040057771555991, + "mae": 0.39488768577575684, + "r2": 0.5238790512084961, + "score": -0.5040057771555991 + }, + "val": { + "rmse": 0.5558749906446844, + "mae": 0.4398687481880188, + "r2": 0.4173172116279602, + "score": -0.5558749906446844 + }, + "test": { + "rmse": 0.5496162918403876, + "mae": 0.44109272956848145, + "r2": 0.4371039867401123, + "score": -0.5496162918403876 + } + }, + "time": "0:09:34.700424", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adapterf-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adapterf-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adapterf-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adapterf-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..353b2b7b7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adapterf-greedy-heads-evaluation/0/report.json @@ -0,0 +1,242 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adapterf-evaluation/_esam_diagnostics/deliv_esam_rho0p001_sf0p5_adapterf-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5042286809765437, + "mae": 0.39605751633644104, + "r2": 0.5234577655792236, + "score": -0.5042286809765437 + }, + "val": { + "rmse": 0.5553348954063525, + "mae": 0.44016024470329285, + "r2": 0.4184489846229553, + "score": -0.5553348954063525 + }, + "test": { + "rmse": 0.550360417400049, + "mae": 0.44305530190467834, + "r2": 0.43557870388031006, + "score": -0.550360417400049 + } + }, + "time": "0:09:34.700424", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13, + 6, + 10, + 14, + 26, + 27, + 18, + 25, + 6, + 15, + 8, + 17, + 10, + 26 + ], + "head_selection_time": "0:00:00.436537" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adaptert-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adaptert-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adaptert-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adaptert-best-head-evaluation/0/report.json new file mode 100644 index 000000000..9a11da961 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adaptert-best-head-evaluation/0/report.json @@ -0,0 +1,178 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adaptert-evaluation/_esam_diagnostics/deliv_esam_rho0p001_sf0p5_adaptert-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5112224530864115, + "mae": 0.40589213371276855, + "r2": 0.5101466178894043, + "score": -0.5112224530864115 + }, + "val": { + "rmse": 0.5583881453279098, + "mae": 0.4439501166343689, + "r2": 0.412036657333374, + "score": -0.5583881453279098 + }, + "test": { + "rmse": 0.5557654209580059, + "mae": 0.4505227506160736, + "r2": 0.42443811893463135, + "score": -0.5557654209580059 + } + }, + "time": "0:09:06.259977", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442888140678406, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958992034196854 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023674" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adaptert-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adaptert-evaluation/0.toml new file mode 100644 index 000000000..79f334591 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adaptert-evaluation/0.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.001 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 16 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adaptert-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adaptert-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adaptert-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adaptert-evaluation/0/report.json new file mode 100644 index 000000000..5a09ad7c7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adaptert-evaluation/0/report.json @@ -0,0 +1,174 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adaptert-evaluation/_esam_diagnostics/deliv_esam_rho0p001_sf0p5_adaptert-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5040057771555991, + "mae": 0.39488768577575684, + "r2": 0.5238790512084961, + "score": -0.5040057771555991 + }, + "val": { + "rmse": 0.5558750174513605, + "mae": 0.4398687481880188, + "r2": 0.41731715202331543, + "score": -0.5558750174513605 + }, + "test": { + "rmse": 0.5496162918403876, + "mae": 0.44109272956848145, + "r2": 0.4371039867401123, + "score": -0.5496162918403876 + } + }, + "time": "0:09:06.259977", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442888140678406, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958992034196854 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adaptert-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adaptert-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adaptert-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adaptert-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..23466521b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adaptert-greedy-heads-evaluation/0/report.json @@ -0,0 +1,191 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adaptert-evaluation/_esam_diagnostics/deliv_esam_rho0p001_sf0p5_adaptert-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.50422871052893, + "mae": 0.39605751633644104, + "r2": 0.5234577655792236, + "score": -0.50422871052893 + }, + "val": { + "rmse": 0.5553348954063525, + "mae": 0.44016024470329285, + "r2": 0.4184489846229553, + "score": -0.5553348954063525 + }, + "test": { + "rmse": 0.550360417400049, + "mae": 0.44305530190467834, + "r2": 0.43557870388031006, + "score": -0.550360417400049 + } + }, + "time": "0:09:06.259977", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442888140678406, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958992034196854 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13, + 6, + 10, + 14, + 26, + 27, + 18, + 25, + 6, + 15, + 8, + 17, + 10, + 26 + ], + "head_selection_time": "0:00:00.420549" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_numerical_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_numerical_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_numerical_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_numerical_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..e12ce29d6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_numerical_only-best-head-evaluation/0/report.json @@ -0,0 +1,212 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" + }, + "mask_stats": { + "keep_rate_actual": 0.9772727489471436, + "min_features_per_member": 210.0, + "mean_features_per_member": 215.0, + "max_features_per_member": 220.0, + "min_members_per_feature": 29.0, + "mean_members_per_feature": 31.272727966308594, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.955511449861677, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.00909090909090909, + "n_core_features": 2.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5081016011029494, + "mae": 0.4003114700317383, + "r2": 0.5161091685295105, + "score": -0.5081016011029494 + }, + "val": { + "rmse": 0.5590051856389011, + "mae": 0.4407220184803009, + "r2": 0.41073644161224365, + "score": -0.5590051856389011 + }, + "test": { + "rmse": 0.5534488118080276, + "mae": 0.4457874596118927, + "r2": 0.4292263984680176, + "score": -0.5534488118080276 + } + }, + "time": "0:07:44.425919", + "diversity": { + "train": { + "mean_centered_corr": -0.02009296789765358, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07790226489305496 + }, + "val": { + "mean_centered_corr": -0.02050519548356533, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07800902426242828 + }, + "test": { + "mean_centered_corr": -0.02043369971215725, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07650945335626602 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 6 + ], + "head_selection_time": "0:00:00.024746" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_numerical_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_numerical_only-evaluation/0.toml new file mode 100644 index 000000000..ba5fa4110 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_numerical_only-evaluation/0.toml @@ -0,0 +1,50 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 +start_epoch = 16 +start_fraction = 0.5 +group_mode = "numerical_only" +categorical_handling = "no_cat_drop" + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_numerical_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_numerical_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_numerical_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_numerical_only-evaluation/0/report.json new file mode 100644 index 000000000..3f118c5a6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_numerical_only-evaluation/0/report.json @@ -0,0 +1,208 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" + }, + "mask_stats": { + "keep_rate_actual": 0.9772727489471436, + "min_features_per_member": 210.0, + "mean_features_per_member": 215.0, + "max_features_per_member": 220.0, + "min_members_per_feature": 29.0, + "mean_members_per_feature": 31.272727966308594, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.955511449861677, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.00909090909090909, + "n_core_features": 2.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5047748974218713, + "mae": 0.39609476923942566, + "r2": 0.5224248170852661, + "score": -0.5047748974218713 + }, + "val": { + "rmse": 0.5558048331442785, + "mae": 0.43993815779685974, + "r2": 0.41746431589126587, + "score": -0.5558048331442785 + }, + "test": { + "rmse": 0.5501619198178428, + "mae": 0.44212156534194946, + "r2": 0.435985803604126, + "score": -0.5501619198178428 + } + }, + "time": "0:07:44.425919", + "diversity": { + "train": { + "mean_centered_corr": -0.02009296789765358, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07790226489305496 + }, + "val": { + "mean_centered_corr": -0.02050519548356533, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07800902426242828 + }, + "test": { + "mean_centered_corr": -0.02043369971215725, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07650945335626602 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_numerical_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_numerical_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_numerical_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_numerical_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..b43e4ade6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_numerical_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,224 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" + }, + "mask_stats": { + "keep_rate_actual": 0.9772727489471436, + "min_features_per_member": 210.0, + "mean_features_per_member": 215.0, + "max_features_per_member": 220.0, + "min_members_per_feature": 29.0, + "mean_members_per_feature": 31.272727966308594, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.955511449861677, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.00909090909090909, + "n_core_features": 2.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5051252716316714, + "mae": 0.39751043915748596, + "r2": 0.5217615962028503, + "score": -0.5051252716316714 + }, + "val": { + "rmse": 0.5553099940581016, + "mae": 0.44019240140914917, + "r2": 0.4185011386871338, + "score": -0.5553099940581016 + }, + "test": { + "rmse": 0.5510631408064661, + "mae": 0.44413861632347107, + "r2": 0.43413645029067993, + "score": -0.5510631408064661 + } + }, + "time": "0:07:44.425919", + "diversity": { + "train": { + "mean_centered_corr": -0.02009296789765358, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07790226489305496 + }, + "val": { + "mean_centered_corr": -0.02050519548356533, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07800902426242828 + }, + "test": { + "mean_centered_corr": -0.02043369971215725, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07650945335626602 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 6, + 13, + 26, + 10, + 14, + 8, + 7, + 27, + 17, + 26, + 13, + 25, + 19 + ], + "head_selection_time": "0:00:00.409868" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_per_member-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_per_member-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_per_member-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_per_member-best-head-evaluation/0/report.json new file mode 100644 index 000000000..2144dbac0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_per_member-best-head-evaluation/0/report.json @@ -0,0 +1,212 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "per_member", + "categorical_handling": "no_cat_drop" + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "per_member", + "categorical_handling": "no_cat_drop" + }, + "mask_stats": { + "keep_rate_actual": 0.9772727489471436, + "min_features_per_member": 210.0, + "mean_features_per_member": 215.0, + "max_features_per_member": 220.0, + "min_members_per_feature": 29.0, + "mean_members_per_feature": 31.272727966308594, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.955511449861677, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.00909090909090909, + "n_core_features": 2.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5081016011029494, + "mae": 0.4003114700317383, + "r2": 0.5161091685295105, + "score": -0.5081016011029494 + }, + "val": { + "rmse": 0.5590051856389011, + "mae": 0.4407220184803009, + "r2": 0.41073644161224365, + "score": -0.5590051856389011 + }, + "test": { + "rmse": 0.5534488118080276, + "mae": 0.4457874596118927, + "r2": 0.4292263984680176, + "score": -0.5534488118080276 + } + }, + "time": "0:07:43.333698", + "diversity": { + "train": { + "mean_centered_corr": -0.02009296789765358, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07790226489305496 + }, + "val": { + "mean_centered_corr": -0.02050519548356533, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07800902426242828 + }, + "test": { + "mean_centered_corr": -0.02043369971215725, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07650945335626602 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 6 + ], + "head_selection_time": "0:00:00.023570" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_per_member-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_per_member-evaluation/0.toml new file mode 100644 index 000000000..feee7a060 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_per_member-evaluation/0.toml @@ -0,0 +1,50 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 +start_epoch = 16 +start_fraction = 0.5 +group_mode = "per_member" +categorical_handling = "no_cat_drop" + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_per_member-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_per_member-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_per_member-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_per_member-evaluation/0/report.json new file mode 100644 index 000000000..c354cbffb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_per_member-evaluation/0/report.json @@ -0,0 +1,208 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "per_member", + "categorical_handling": "no_cat_drop" + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "per_member", + "categorical_handling": "no_cat_drop" + }, + "mask_stats": { + "keep_rate_actual": 0.9772727489471436, + "min_features_per_member": 210.0, + "mean_features_per_member": 215.0, + "max_features_per_member": 220.0, + "min_members_per_feature": 29.0, + "mean_members_per_feature": 31.272727966308594, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.955511449861677, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.00909090909090909, + "n_core_features": 2.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5047748974218713, + "mae": 0.39609476923942566, + "r2": 0.5224248170852661, + "score": -0.5047748974218713 + }, + "val": { + "rmse": 0.5558048331442785, + "mae": 0.43993815779685974, + "r2": 0.41746431589126587, + "score": -0.5558048331442785 + }, + "test": { + "rmse": 0.5501619198178428, + "mae": 0.44212156534194946, + "r2": 0.435985803604126, + "score": -0.5501619198178428 + } + }, + "time": "0:07:43.333698", + "diversity": { + "train": { + "mean_centered_corr": -0.02009296789765358, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07790226489305496 + }, + "val": { + "mean_centered_corr": -0.02050519548356533, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07800902426242828 + }, + "test": { + "mean_centered_corr": -0.02043369971215725, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07650945335626602 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_per_member-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_per_member-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_per_member-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_per_member-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..bb3c18a92 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_per_member-greedy-heads-evaluation/0/report.json @@ -0,0 +1,224 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "per_member", + "categorical_handling": "no_cat_drop" + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "per_member", + "categorical_handling": "no_cat_drop" + }, + "mask_stats": { + "keep_rate_actual": 0.9772727489471436, + "min_features_per_member": 210.0, + "mean_features_per_member": 215.0, + "max_features_per_member": 220.0, + "min_members_per_feature": 29.0, + "mean_members_per_feature": 31.272727966308594, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.955511449861677, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.00909090909090909, + "n_core_features": 2.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5051252716316714, + "mae": 0.39751043915748596, + "r2": 0.5217615962028503, + "score": -0.5051252716316714 + }, + "val": { + "rmse": 0.5553099940581016, + "mae": 0.44019240140914917, + "r2": 0.4185011386871338, + "score": -0.5553099940581016 + }, + "test": { + "rmse": 0.5510631408064661, + "mae": 0.44413861632347107, + "r2": 0.43413645029067993, + "score": -0.5510631408064661 + } + }, + "time": "0:07:43.333698", + "diversity": { + "train": { + "mean_centered_corr": -0.02009296789765358, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07790226489305496 + }, + "val": { + "mean_centered_corr": -0.02050519548356533, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07800902426242828 + }, + "test": { + "mean_centered_corr": -0.02043369971215725, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07650945335626602 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 6, + 13, + 26, + 10, + 14, + 8, + 7, + 27, + 17, + 26, + 13, + 25, + 19 + ], + "head_selection_time": "0:00:00.397135" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_numerical_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_numerical_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_numerical_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_numerical_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..e90bc20e9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_numerical_only-best-head-evaluation/0/report.json @@ -0,0 +1,212 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 24, + "start_fraction": 0.75, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 24, + "start_fraction": 0.75, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" + }, + "mask_stats": { + "keep_rate_actual": 0.9772727489471436, + "min_features_per_member": 210.0, + "mean_features_per_member": 215.0, + "max_features_per_member": 220.0, + "min_members_per_feature": 29.0, + "mean_members_per_feature": 31.272727966308594, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.955511449861677, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.00909090909090909, + "n_core_features": 2.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5081016011029494, + "mae": 0.4003114700317383, + "r2": 0.5161091685295105, + "score": -0.5081016011029494 + }, + "val": { + "rmse": 0.5590051856389011, + "mae": 0.4407220184803009, + "r2": 0.41073644161224365, + "score": -0.5590051856389011 + }, + "test": { + "rmse": 0.5534488118080276, + "mae": 0.4457874596118927, + "r2": 0.4292263984680176, + "score": -0.5534488118080276 + } + }, + "time": "0:07:18.478027", + "diversity": { + "train": { + "mean_centered_corr": -0.02009296789765358, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07790226489305496 + }, + "val": { + "mean_centered_corr": -0.02050519548356533, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07800902426242828 + }, + "test": { + "mean_centered_corr": -0.02043369971215725, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07650945335626602 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 6 + ], + "head_selection_time": "0:00:00.023345" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_numerical_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_numerical_only-evaluation/0.toml new file mode 100644 index 000000000..e61f34d87 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_numerical_only-evaluation/0.toml @@ -0,0 +1,50 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 +start_epoch = 24 +start_fraction = 0.75 +group_mode = "numerical_only" +categorical_handling = "no_cat_drop" + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_numerical_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_numerical_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_numerical_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_numerical_only-evaluation/0/report.json new file mode 100644 index 000000000..2ffadf096 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_numerical_only-evaluation/0/report.json @@ -0,0 +1,208 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 24, + "start_fraction": 0.75, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 24, + "start_fraction": 0.75, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" + }, + "mask_stats": { + "keep_rate_actual": 0.9772727489471436, + "min_features_per_member": 210.0, + "mean_features_per_member": 215.0, + "max_features_per_member": 220.0, + "min_members_per_feature": 29.0, + "mean_members_per_feature": 31.272727966308594, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.955511449861677, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.00909090909090909, + "n_core_features": 2.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5047748974218713, + "mae": 0.39609476923942566, + "r2": 0.5224248170852661, + "score": -0.5047748974218713 + }, + "val": { + "rmse": 0.5558048331442785, + "mae": 0.43993815779685974, + "r2": 0.41746431589126587, + "score": -0.5558048331442785 + }, + "test": { + "rmse": 0.5501619198178428, + "mae": 0.44212156534194946, + "r2": 0.435985803604126, + "score": -0.5501619198178428 + } + }, + "time": "0:07:18.478027", + "diversity": { + "train": { + "mean_centered_corr": -0.02009296789765358, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07790226489305496 + }, + "val": { + "mean_centered_corr": -0.02050519548356533, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07800902426242828 + }, + "test": { + "mean_centered_corr": -0.02043369971215725, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07650945335626602 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_numerical_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_numerical_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_numerical_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_numerical_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..05a9734ef --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_numerical_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,224 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 24, + "start_fraction": 0.75, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 24, + "start_fraction": 0.75, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" + }, + "mask_stats": { + "keep_rate_actual": 0.9772727489471436, + "min_features_per_member": 210.0, + "mean_features_per_member": 215.0, + "max_features_per_member": 220.0, + "min_members_per_feature": 29.0, + "mean_members_per_feature": 31.272727966308594, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.955511449861677, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.00909090909090909, + "n_core_features": 2.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5051252716316714, + "mae": 0.39751043915748596, + "r2": 0.5217615962028503, + "score": -0.5051252716316714 + }, + "val": { + "rmse": 0.5553099940581016, + "mae": 0.44019240140914917, + "r2": 0.4185011386871338, + "score": -0.5553099940581016 + }, + "test": { + "rmse": 0.5510631408064661, + "mae": 0.44413861632347107, + "r2": 0.43413645029067993, + "score": -0.5510631408064661 + } + }, + "time": "0:07:18.478027", + "diversity": { + "train": { + "mean_centered_corr": -0.02009296789765358, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07790226489305496 + }, + "val": { + "mean_centered_corr": -0.02050519548356533, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07800902426242828 + }, + "test": { + "mean_centered_corr": -0.02043369971215725, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07650945335626602 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 6, + 13, + 26, + 10, + 14, + 8, + 7, + 27, + 17, + 26, + 13, + 25, + 19 + ], + "head_selection_time": "0:00:00.396856" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_per_member-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_per_member-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_per_member-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_per_member-best-head-evaluation/0/report.json new file mode 100644 index 000000000..c5f593cdf --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_per_member-best-head-evaluation/0/report.json @@ -0,0 +1,212 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 24, + "start_fraction": 0.75, + "group_mode": "per_member", + "categorical_handling": "no_cat_drop" + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 24, + "start_fraction": 0.75, + "group_mode": "per_member", + "categorical_handling": "no_cat_drop" + }, + "mask_stats": { + "keep_rate_actual": 0.9772727489471436, + "min_features_per_member": 210.0, + "mean_features_per_member": 215.0, + "max_features_per_member": 220.0, + "min_members_per_feature": 29.0, + "mean_members_per_feature": 31.272727966308594, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.955511449861677, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.00909090909090909, + "n_core_features": 2.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5081016011029494, + "mae": 0.4003114700317383, + "r2": 0.5161091685295105, + "score": -0.5081016011029494 + }, + "val": { + "rmse": 0.5590051856389011, + "mae": 0.4407220184803009, + "r2": 0.41073644161224365, + "score": -0.5590051856389011 + }, + "test": { + "rmse": 0.5534488118080276, + "mae": 0.4457874596118927, + "r2": 0.4292263984680176, + "score": -0.5534488118080276 + } + }, + "time": "0:07:19.574103", + "diversity": { + "train": { + "mean_centered_corr": -0.02009296789765358, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07790226489305496 + }, + "val": { + "mean_centered_corr": -0.02050519548356533, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07800902426242828 + }, + "test": { + "mean_centered_corr": -0.02043369971215725, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07650945335626602 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 6 + ], + "head_selection_time": "0:00:00.023618" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_per_member-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_per_member-evaluation/0.toml new file mode 100644 index 000000000..636b0362e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_per_member-evaluation/0.toml @@ -0,0 +1,50 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 +start_epoch = 24 +start_fraction = 0.75 +group_mode = "per_member" +categorical_handling = "no_cat_drop" + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_per_member-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_per_member-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_per_member-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_per_member-evaluation/0/report.json new file mode 100644 index 000000000..25addf8bd --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_per_member-evaluation/0/report.json @@ -0,0 +1,208 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 24, + "start_fraction": 0.75, + "group_mode": "per_member", + "categorical_handling": "no_cat_drop" + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 24, + "start_fraction": 0.75, + "group_mode": "per_member", + "categorical_handling": "no_cat_drop" + }, + "mask_stats": { + "keep_rate_actual": 0.9772727489471436, + "min_features_per_member": 210.0, + "mean_features_per_member": 215.0, + "max_features_per_member": 220.0, + "min_members_per_feature": 29.0, + "mean_members_per_feature": 31.272727966308594, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.955511449861677, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.00909090909090909, + "n_core_features": 2.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5047748974218713, + "mae": 0.39609476923942566, + "r2": 0.5224248170852661, + "score": -0.5047748974218713 + }, + "val": { + "rmse": 0.5558048331442785, + "mae": 0.43993815779685974, + "r2": 0.41746431589126587, + "score": -0.5558048331442785 + }, + "test": { + "rmse": 0.5501619198178428, + "mae": 0.44212156534194946, + "r2": 0.435985803604126, + "score": -0.5501619198178428 + } + }, + "time": "0:07:19.574103", + "diversity": { + "train": { + "mean_centered_corr": -0.02009296789765358, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07790226489305496 + }, + "val": { + "mean_centered_corr": -0.02050519548356533, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07800902426242828 + }, + "test": { + "mean_centered_corr": -0.02043369971215725, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07650945335626602 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_per_member-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_per_member-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_per_member-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_per_member-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..7683a0567 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_per_member-greedy-heads-evaluation/0/report.json @@ -0,0 +1,224 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 24, + "start_fraction": 0.75, + "group_mode": "per_member", + "categorical_handling": "no_cat_drop" + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 24, + "start_fraction": 0.75, + "group_mode": "per_member", + "categorical_handling": "no_cat_drop" + }, + "mask_stats": { + "keep_rate_actual": 0.9772727489471436, + "min_features_per_member": 210.0, + "mean_features_per_member": 215.0, + "max_features_per_member": 220.0, + "min_members_per_feature": 29.0, + "mean_members_per_feature": 31.272727966308594, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.955511449861677, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.00909090909090909, + "n_core_features": 2.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5051252716316714, + "mae": 0.39751043915748596, + "r2": 0.5217615962028503, + "score": -0.5051252716316714 + }, + "val": { + "rmse": 0.5553099940581016, + "mae": 0.44019240140914917, + "r2": 0.4185011386871338, + "score": -0.5553099940581016 + }, + "test": { + "rmse": 0.5510631408064661, + "mae": 0.44413861632347107, + "r2": 0.43413645029067993, + "score": -0.5510631408064661 + } + }, + "time": "0:07:19.574103", + "diversity": { + "train": { + "mean_centered_corr": -0.02009296789765358, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07790226489305496 + }, + "val": { + "mean_centered_corr": -0.02050519548356533, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07800902426242828 + }, + "test": { + "mean_centered_corr": -0.02043369971215725, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07650945335626602 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 6, + 13, + 26, + 10, + 14, + 8, + 7, + 27, + 17, + 26, + 13, + 25, + 19 + ], + "head_selection_time": "0:00:00.395432" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_numerical_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_numerical_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_numerical_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_numerical_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..bde2c36a5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_numerical_only-best-head-evaluation/0/report.json @@ -0,0 +1,212 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.99, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.99, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" + }, + "mask_stats": { + "keep_rate_actual": 0.9903408885002136, + "min_features_per_member": 213.0, + "mean_features_per_member": 217.875, + "max_features_per_member": 220.0, + "min_members_per_feature": 30.0, + "mean_members_per_feature": 31.690908432006836, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9807907427121413, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.00909090909090909, + "n_core_features": 2.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5111845008565297, + "mae": 0.4058074951171875, + "r2": 0.5102193355560303, + "score": -0.5111845008565297 + }, + "val": { + "rmse": 0.5584120821811833, + "mae": 0.44400346279144287, + "r2": 0.41198623180389404, + "score": -0.5584120821811833 + }, + "test": { + "rmse": 0.5559629363941286, + "mae": 0.4507065415382385, + "r2": 0.4240289330482483, + "score": -0.5559629363941286 + } + }, + "time": "0:07:44.999332", + "diversity": { + "train": { + "mean_centered_corr": -0.02024354599416256, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07271759957075119 + }, + "val": { + "mean_centered_corr": -0.02031630650162697, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07322525978088379 + }, + "test": { + "mean_centered_corr": -0.02031359262764454, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07238791882991791 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023353" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_numerical_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_numerical_only-evaluation/0.toml new file mode 100644 index 000000000..8b577989a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_numerical_only-evaluation/0.toml @@ -0,0 +1,50 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.99 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 +start_epoch = 16 +start_fraction = 0.5 +group_mode = "numerical_only" +categorical_handling = "no_cat_drop" + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_numerical_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_numerical_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_numerical_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_numerical_only-evaluation/0/report.json new file mode 100644 index 000000000..235a79e5d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_numerical_only-evaluation/0/report.json @@ -0,0 +1,208 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.99, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.99, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" + }, + "mask_stats": { + "keep_rate_actual": 0.9903408885002136, + "min_features_per_member": 213.0, + "mean_features_per_member": 217.875, + "max_features_per_member": 220.0, + "min_members_per_feature": 30.0, + "mean_members_per_feature": 31.690908432006836, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9807907427121413, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.00909090909090909, + "n_core_features": 2.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5042872799552254, + "mae": 0.39527493715286255, + "r2": 0.523347020149231, + "score": -0.5042872799552254 + }, + "val": { + "rmse": 0.5558106240871751, + "mae": 0.4399650990962982, + "r2": 0.4174521565437317, + "score": -0.5558106240871751 + }, + "test": { + "rmse": 0.5498568034160569, + "mae": 0.44145897030830383, + "r2": 0.43661123514175415, + "score": -0.5498568034160569 + } + }, + "time": "0:07:44.999332", + "diversity": { + "train": { + "mean_centered_corr": -0.02024354599416256, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07271759957075119 + }, + "val": { + "mean_centered_corr": -0.02031630650162697, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07322525978088379 + }, + "test": { + "mean_centered_corr": -0.02031359262764454, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07238791882991791 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_numerical_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_numerical_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_numerical_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_numerical_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..805f8d13c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_numerical_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,224 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.99, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.99, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" + }, + "mask_stats": { + "keep_rate_actual": 0.9903408885002136, + "min_features_per_member": 213.0, + "mean_features_per_member": 217.875, + "max_features_per_member": 220.0, + "min_members_per_feature": 30.0, + "mean_members_per_feature": 31.690908432006836, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9807907427121413, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.00909090909090909, + "n_core_features": 2.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5048514675541059, + "mae": 0.397201806306839, + "r2": 0.5222799181938171, + "score": -0.5048514675541059 + }, + "val": { + "rmse": 0.5553168366734594, + "mae": 0.4400327205657959, + "r2": 0.4184867739677429, + "score": -0.5553168366734594 + }, + "test": { + "rmse": 0.5509050649441617, + "mae": 0.44378286600112915, + "r2": 0.4344610571861267, + "score": -0.5509050649441617 + } + }, + "time": "0:07:44.999332", + "diversity": { + "train": { + "mean_centered_corr": -0.02024354599416256, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07271759957075119 + }, + "val": { + "mean_centered_corr": -0.02031630650162697, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07322525978088379 + }, + "test": { + "mean_centered_corr": -0.02031359262764454, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07238791882991791 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13, + 6, + 14, + 10, + 26, + 27, + 7, + 6, + 29, + 8, + 15, + 17, + 6 + ], + "head_selection_time": "0:00:00.394414" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_per_member-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_per_member-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_per_member-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_per_member-best-head-evaluation/0/report.json new file mode 100644 index 000000000..1886d86c0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_per_member-best-head-evaluation/0/report.json @@ -0,0 +1,212 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.99, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "per_member", + "categorical_handling": "no_cat_drop" + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.99, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "per_member", + "categorical_handling": "no_cat_drop" + }, + "mask_stats": { + "keep_rate_actual": 0.9903408885002136, + "min_features_per_member": 213.0, + "mean_features_per_member": 217.875, + "max_features_per_member": 220.0, + "min_members_per_feature": 30.0, + "mean_members_per_feature": 31.690908432006836, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9807907427121413, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.00909090909090909, + "n_core_features": 2.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5111845008565297, + "mae": 0.4058074951171875, + "r2": 0.5102193355560303, + "score": -0.5111845008565297 + }, + "val": { + "rmse": 0.5584120821811833, + "mae": 0.44400346279144287, + "r2": 0.41198623180389404, + "score": -0.5584120821811833 + }, + "test": { + "rmse": 0.5559629363941286, + "mae": 0.4507065415382385, + "r2": 0.4240289330482483, + "score": -0.5559629363941286 + } + }, + "time": "0:07:44.772554", + "diversity": { + "train": { + "mean_centered_corr": -0.02024354599416256, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07271759957075119 + }, + "val": { + "mean_centered_corr": -0.02031630650162697, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07322525978088379 + }, + "test": { + "mean_centered_corr": -0.02031359262764454, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07238791882991791 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023456" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_per_member-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_per_member-evaluation/0.toml new file mode 100644 index 000000000..08a257a0b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_per_member-evaluation/0.toml @@ -0,0 +1,50 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.99 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 +start_epoch = 16 +start_fraction = 0.5 +group_mode = "per_member" +categorical_handling = "no_cat_drop" + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_per_member-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_per_member-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_per_member-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_per_member-evaluation/0/report.json new file mode 100644 index 000000000..d0a942f0b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_per_member-evaluation/0/report.json @@ -0,0 +1,208 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.99, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "per_member", + "categorical_handling": "no_cat_drop" + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.99, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "per_member", + "categorical_handling": "no_cat_drop" + }, + "mask_stats": { + "keep_rate_actual": 0.9903408885002136, + "min_features_per_member": 213.0, + "mean_features_per_member": 217.875, + "max_features_per_member": 220.0, + "min_members_per_feature": 30.0, + "mean_members_per_feature": 31.690908432006836, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9807907427121413, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.00909090909090909, + "n_core_features": 2.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5042872799552254, + "mae": 0.39527493715286255, + "r2": 0.523347020149231, + "score": -0.5042872799552254 + }, + "val": { + "rmse": 0.5558106240871751, + "mae": 0.4399650990962982, + "r2": 0.4174521565437317, + "score": -0.5558106240871751 + }, + "test": { + "rmse": 0.5498568034160569, + "mae": 0.44145897030830383, + "r2": 0.43661123514175415, + "score": -0.5498568034160569 + } + }, + "time": "0:07:44.772554", + "diversity": { + "train": { + "mean_centered_corr": -0.02024354599416256, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07271759957075119 + }, + "val": { + "mean_centered_corr": -0.02031630650162697, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07322525978088379 + }, + "test": { + "mean_centered_corr": -0.02031359262764454, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07238791882991791 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_per_member-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_per_member-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_per_member-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_per_member-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..cb8c389b1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_per_member-greedy-heads-evaluation/0/report.json @@ -0,0 +1,224 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.99, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "per_member", + "categorical_handling": "no_cat_drop" + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.99, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "per_member", + "categorical_handling": "no_cat_drop" + }, + "mask_stats": { + "keep_rate_actual": 0.9903408885002136, + "min_features_per_member": 213.0, + "mean_features_per_member": 217.875, + "max_features_per_member": 220.0, + "min_members_per_feature": 30.0, + "mean_members_per_feature": 31.690908432006836, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9807907427121413, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.00909090909090909, + "n_core_features": 2.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5048514675541059, + "mae": 0.397201806306839, + "r2": 0.5222799181938171, + "score": -0.5048514675541059 + }, + "val": { + "rmse": 0.5553168366734594, + "mae": 0.4400327205657959, + "r2": 0.4184867739677429, + "score": -0.5553168366734594 + }, + "test": { + "rmse": 0.5509050649441617, + "mae": 0.44378286600112915, + "r2": 0.4344610571861267, + "score": -0.5509050649441617 + } + }, + "time": "0:07:44.772554", + "diversity": { + "train": { + "mean_centered_corr": -0.02024354599416256, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07271759957075119 + }, + "val": { + "mean_centered_corr": -0.02031630650162697, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07322525978088379 + }, + "test": { + "mean_centered_corr": -0.02031359262764454, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07238791882991791 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13, + 6, + 14, + 10, + 26, + 27, + 7, + 6, + 29, + 8, + 15, + 17, + 6 + ], + "head_selection_time": "0:00:00.400193" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_numerical_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_numerical_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_numerical_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_numerical_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..4b7f7549e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_numerical_only-best-head-evaluation/0/report.json @@ -0,0 +1,212 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.99, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 24, + "start_fraction": 0.75, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.99, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 24, + "start_fraction": 0.75, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" + }, + "mask_stats": { + "keep_rate_actual": 0.9903408885002136, + "min_features_per_member": 213.0, + "mean_features_per_member": 217.875, + "max_features_per_member": 220.0, + "min_members_per_feature": 30.0, + "mean_members_per_feature": 31.690908432006836, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9807907427121413, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.00909090909090909, + "n_core_features": 2.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5111845008565297, + "mae": 0.4058074951171875, + "r2": 0.5102193355560303, + "score": -0.5111845008565297 + }, + "val": { + "rmse": 0.5584120821811833, + "mae": 0.44400346279144287, + "r2": 0.41198623180389404, + "score": -0.5584120821811833 + }, + "test": { + "rmse": 0.5559629363941286, + "mae": 0.4507065415382385, + "r2": 0.4240289330482483, + "score": -0.5559629363941286 + } + }, + "time": "0:07:21.324075", + "diversity": { + "train": { + "mean_centered_corr": -0.02024354599416256, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07271759957075119 + }, + "val": { + "mean_centered_corr": -0.02031630650162697, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07322525978088379 + }, + "test": { + "mean_centered_corr": -0.02031359262764454, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07238791882991791 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023316" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_numerical_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_numerical_only-evaluation/0.toml new file mode 100644 index 000000000..13878a19d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_numerical_only-evaluation/0.toml @@ -0,0 +1,50 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.99 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 +start_epoch = 24 +start_fraction = 0.75 +group_mode = "numerical_only" +categorical_handling = "no_cat_drop" + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_numerical_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_numerical_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_numerical_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_numerical_only-evaluation/0/report.json new file mode 100644 index 000000000..501bd3a58 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_numerical_only-evaluation/0/report.json @@ -0,0 +1,208 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.99, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 24, + "start_fraction": 0.75, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.99, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 24, + "start_fraction": 0.75, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" + }, + "mask_stats": { + "keep_rate_actual": 0.9903408885002136, + "min_features_per_member": 213.0, + "mean_features_per_member": 217.875, + "max_features_per_member": 220.0, + "min_members_per_feature": 30.0, + "mean_members_per_feature": 31.690908432006836, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9807907427121413, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.00909090909090909, + "n_core_features": 2.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5042872799552254, + "mae": 0.39527493715286255, + "r2": 0.523347020149231, + "score": -0.5042872799552254 + }, + "val": { + "rmse": 0.5558106240871751, + "mae": 0.4399650990962982, + "r2": 0.4174521565437317, + "score": -0.5558106240871751 + }, + "test": { + "rmse": 0.5498568034160569, + "mae": 0.44145897030830383, + "r2": 0.43661123514175415, + "score": -0.5498568034160569 + } + }, + "time": "0:07:21.324075", + "diversity": { + "train": { + "mean_centered_corr": -0.02024354599416256, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07271759957075119 + }, + "val": { + "mean_centered_corr": -0.02031630650162697, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07322525978088379 + }, + "test": { + "mean_centered_corr": -0.02031359262764454, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07238791882991791 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_numerical_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_numerical_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_numerical_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_numerical_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..39b35ad63 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_numerical_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,224 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.99, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 24, + "start_fraction": 0.75, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.99, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 24, + "start_fraction": 0.75, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" + }, + "mask_stats": { + "keep_rate_actual": 0.9903408885002136, + "min_features_per_member": 213.0, + "mean_features_per_member": 217.875, + "max_features_per_member": 220.0, + "min_members_per_feature": 30.0, + "mean_members_per_feature": 31.690908432006836, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9807907427121413, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.00909090909090909, + "n_core_features": 2.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5048514675541059, + "mae": 0.397201806306839, + "r2": 0.5222799181938171, + "score": -0.5048514675541059 + }, + "val": { + "rmse": 0.5553168366734594, + "mae": 0.4400327205657959, + "r2": 0.4184867739677429, + "score": -0.5553168366734594 + }, + "test": { + "rmse": 0.5509050649441617, + "mae": 0.44378286600112915, + "r2": 0.4344610571861267, + "score": -0.5509050649441617 + } + }, + "time": "0:07:21.324075", + "diversity": { + "train": { + "mean_centered_corr": -0.02024354599416256, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07271759957075119 + }, + "val": { + "mean_centered_corr": -0.02031630650162697, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07322525978088379 + }, + "test": { + "mean_centered_corr": -0.02031359262764454, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07238791882991791 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13, + 6, + 14, + 10, + 26, + 27, + 7, + 6, + 29, + 8, + 15, + 17, + 6 + ], + "head_selection_time": "0:00:00.396168" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_per_member-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_per_member-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_per_member-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_per_member-best-head-evaluation/0/report.json new file mode 100644 index 000000000..4fa13f71b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_per_member-best-head-evaluation/0/report.json @@ -0,0 +1,212 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.99, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 24, + "start_fraction": 0.75, + "group_mode": "per_member", + "categorical_handling": "no_cat_drop" + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.99, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 24, + "start_fraction": 0.75, + "group_mode": "per_member", + "categorical_handling": "no_cat_drop" + }, + "mask_stats": { + "keep_rate_actual": 0.9903408885002136, + "min_features_per_member": 213.0, + "mean_features_per_member": 217.875, + "max_features_per_member": 220.0, + "min_members_per_feature": 30.0, + "mean_members_per_feature": 31.690908432006836, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9807907427121413, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.00909090909090909, + "n_core_features": 2.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5111845008565297, + "mae": 0.4058074951171875, + "r2": 0.5102193355560303, + "score": -0.5111845008565297 + }, + "val": { + "rmse": 0.5584120821811833, + "mae": 0.44400346279144287, + "r2": 0.41198623180389404, + "score": -0.5584120821811833 + }, + "test": { + "rmse": 0.5559629363941286, + "mae": 0.4507065415382385, + "r2": 0.4240289330482483, + "score": -0.5559629363941286 + } + }, + "time": "0:07:20.229776", + "diversity": { + "train": { + "mean_centered_corr": -0.02024354599416256, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07271759957075119 + }, + "val": { + "mean_centered_corr": -0.02031630650162697, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07322525978088379 + }, + "test": { + "mean_centered_corr": -0.02031359262764454, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07238791882991791 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023284" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_per_member-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_per_member-evaluation/0.toml new file mode 100644 index 000000000..44f21d6df --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_per_member-evaluation/0.toml @@ -0,0 +1,50 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.99 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 +start_epoch = 24 +start_fraction = 0.75 +group_mode = "per_member" +categorical_handling = "no_cat_drop" + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_per_member-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_per_member-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_per_member-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_per_member-evaluation/0/report.json new file mode 100644 index 000000000..6dbbb8057 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_per_member-evaluation/0/report.json @@ -0,0 +1,208 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.99, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 24, + "start_fraction": 0.75, + "group_mode": "per_member", + "categorical_handling": "no_cat_drop" + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.99, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 24, + "start_fraction": 0.75, + "group_mode": "per_member", + "categorical_handling": "no_cat_drop" + }, + "mask_stats": { + "keep_rate_actual": 0.9903408885002136, + "min_features_per_member": 213.0, + "mean_features_per_member": 217.875, + "max_features_per_member": 220.0, + "min_members_per_feature": 30.0, + "mean_members_per_feature": 31.690908432006836, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9807907427121413, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.00909090909090909, + "n_core_features": 2.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5042872799552254, + "mae": 0.39527493715286255, + "r2": 0.523347020149231, + "score": -0.5042872799552254 + }, + "val": { + "rmse": 0.5558106240871751, + "mae": 0.4399650990962982, + "r2": 0.4174521565437317, + "score": -0.5558106240871751 + }, + "test": { + "rmse": 0.5498568034160569, + "mae": 0.44145897030830383, + "r2": 0.43661123514175415, + "score": -0.5498568034160569 + } + }, + "time": "0:07:20.229776", + "diversity": { + "train": { + "mean_centered_corr": -0.02024354599416256, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07271759957075119 + }, + "val": { + "mean_centered_corr": -0.02031630650162697, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07322525978088379 + }, + "test": { + "mean_centered_corr": -0.02031359262764454, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07238791882991791 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_per_member-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_per_member-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_per_member-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_per_member-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..312a01d8a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_per_member-greedy-heads-evaluation/0/report.json @@ -0,0 +1,224 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.99, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 24, + "start_fraction": 0.75, + "group_mode": "per_member", + "categorical_handling": "no_cat_drop" + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.99, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 24, + "start_fraction": 0.75, + "group_mode": "per_member", + "categorical_handling": "no_cat_drop" + }, + "mask_stats": { + "keep_rate_actual": 0.9903408885002136, + "min_features_per_member": 213.0, + "mean_features_per_member": 217.875, + "max_features_per_member": 220.0, + "min_members_per_feature": 30.0, + "mean_members_per_feature": 31.690908432006836, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9807907427121413, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.00909090909090909, + "n_core_features": 2.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5048514675541059, + "mae": 0.397201806306839, + "r2": 0.5222799181938171, + "score": -0.5048514675541059 + }, + "val": { + "rmse": 0.5553168366734594, + "mae": 0.4400327205657959, + "r2": 0.4184867739677429, + "score": -0.5553168366734594 + }, + "test": { + "rmse": 0.5509050649441617, + "mae": 0.44378286600112915, + "r2": 0.4344610571861267, + "score": -0.5509050649441617 + } + }, + "time": "0:07:20.229776", + "diversity": { + "train": { + "mean_centered_corr": -0.02024354599416256, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07271759957075119 + }, + "val": { + "mean_centered_corr": -0.02031630650162697, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07322525978088379 + }, + "test": { + "mean_centered_corr": -0.02031359262764454, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07238791882991791 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13, + 6, + 14, + 10, + 26, + 27, + 7, + 6, + 29, + 8, + 15, + 17, + 6 + ], + "head_selection_time": "0:00:00.394302" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0-best-head-evaluation/0/report.json new file mode 100644 index 000000000..f195dc2f3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0-best-head-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5112224822345067, + "mae": 0.40589213371276855, + "r2": 0.5101465582847595, + "score": -0.5112224822345067 + }, + "val": { + "rmse": 0.5583881453279098, + "mae": 0.4439501166343689, + "r2": 0.412036657333374, + "score": -0.5583881453279098 + }, + "test": { + "rmse": 0.5557654209580059, + "mae": 0.4505227506160736, + "r2": 0.42443811893463135, + "score": -0.5557654209580059 + } + }, + "time": "0:07:16.610424", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.024247" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0-evaluation/0.toml new file mode 100644 index 000000000..437092c8a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.25 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0-evaluation/0/report.json new file mode 100644 index 000000000..862d22692 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0-evaluation/0/report.json @@ -0,0 +1,171 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5040057771555991, + "mae": 0.39488768577575684, + "r2": 0.5238790512084961, + "score": -0.5040057771555991 + }, + "val": { + "rmse": 0.5558749906446844, + "mae": 0.4398687481880188, + "r2": 0.4173172116279602, + "score": -0.5558749906446844 + }, + "test": { + "rmse": 0.5496162918403876, + "mae": 0.44109272956848145, + "r2": 0.4371039867401123, + "score": -0.5496162918403876 + } + }, + "time": "0:07:16.610424", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..94b76121e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0-greedy-heads-evaluation/0/report.json @@ -0,0 +1,188 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5042286809765437, + "mae": 0.39605751633644104, + "r2": 0.5234577655792236, + "score": -0.5042286809765437 + }, + "val": { + "rmse": 0.5553348954063525, + "mae": 0.44016024470329285, + "r2": 0.4184489846229553, + "score": -0.5553348954063525 + }, + "test": { + "rmse": 0.550360417400049, + "mae": 0.44305530190467834, + "r2": 0.43557870388031006, + "score": -0.550360417400049 + } + }, + "time": "0:07:16.610424", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13, + 6, + 10, + 14, + 26, + 27, + 18, + 25, + 6, + 15, + 8, + 17, + 10, + 26 + ], + "head_selection_time": "0:00:00.455776" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0p5-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0p5-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0p5-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0p5-best-head-evaluation/0/report.json new file mode 100644 index 000000000..61809e291 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0p5-best-head-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5112224822345067, + "mae": 0.40589213371276855, + "r2": 0.5101465582847595, + "score": -0.5112224822345067 + }, + "val": { + "rmse": 0.5583881453279098, + "mae": 0.4439501166343689, + "r2": 0.412036657333374, + "score": -0.5583881453279098 + }, + "test": { + "rmse": 0.5557654209580059, + "mae": 0.4505227506160736, + "r2": 0.42443811893463135, + "score": -0.5557654209580059 + } + }, + "time": "0:07:17.563806", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.025046" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0p5-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0p5-evaluation/0.toml new file mode 100644 index 000000000..683fcc68f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0p5-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.25 +rla_extra_paths_freeze_fraction = 0.5 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0p5-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0p5-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0p5-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0p5-evaluation/0/report.json new file mode 100644 index 000000000..b7563688f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0p5-evaluation/0/report.json @@ -0,0 +1,171 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5040057771555991, + "mae": 0.39488768577575684, + "r2": 0.5238790512084961, + "score": -0.5040057771555991 + }, + "val": { + "rmse": 0.5558749906446844, + "mae": 0.4398687481880188, + "r2": 0.4173172116279602, + "score": -0.5558749906446844 + }, + "test": { + "rmse": 0.5496162918403876, + "mae": 0.44109272956848145, + "r2": 0.4371039867401123, + "score": -0.5496162918403876 + } + }, + "time": "0:07:17.563806", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0p5-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0p5-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0p5-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0p5-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..db5c58c9a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0p5-greedy-heads-evaluation/0/report.json @@ -0,0 +1,188 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5042286809765437, + "mae": 0.39605751633644104, + "r2": 0.5234577655792236, + "score": -0.5042286809765437 + }, + "val": { + "rmse": 0.5553348954063525, + "mae": 0.44016024470329285, + "r2": 0.4184489846229553, + "score": -0.5553348954063525 + }, + "test": { + "rmse": 0.550360417400049, + "mae": 0.44305530190467834, + "r2": 0.43557870388031006, + "score": -0.550360417400049 + } + }, + "time": "0:07:17.563806", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13, + 6, + 10, + 14, + 26, + 27, + 18, + 25, + 6, + 15, + 8, + 17, + 10, + 26 + ], + "head_selection_time": "0:00:00.459154" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0-best-head-evaluation/0/report.json new file mode 100644 index 000000000..7ac826b41 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0-best-head-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5112224822345067, + "mae": 0.40589213371276855, + "r2": 0.5101465582847595, + "score": -0.5112224822345067 + }, + "val": { + "rmse": 0.5583881453279098, + "mae": 0.4439501166343689, + "r2": 0.412036657333374, + "score": -0.5583881453279098 + }, + "test": { + "rmse": 0.5557654209580059, + "mae": 0.4505227506160736, + "r2": 0.42443811893463135, + "score": -0.5557654209580059 + } + }, + "time": "0:07:15.282501", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023342" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0-evaluation/0.toml new file mode 100644 index 000000000..fad5aa191 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.5 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0-evaluation/0/report.json new file mode 100644 index 000000000..2eea74d43 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0-evaluation/0/report.json @@ -0,0 +1,171 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5040057771555991, + "mae": 0.39488768577575684, + "r2": 0.5238790512084961, + "score": -0.5040057771555991 + }, + "val": { + "rmse": 0.5558749906446844, + "mae": 0.4398687481880188, + "r2": 0.4173172116279602, + "score": -0.5558749906446844 + }, + "test": { + "rmse": 0.5496162918403876, + "mae": 0.44109272956848145, + "r2": 0.4371039867401123, + "score": -0.5496162918403876 + } + }, + "time": "0:07:15.282501", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..6f94ed1c7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0-greedy-heads-evaluation/0/report.json @@ -0,0 +1,188 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5042286809765437, + "mae": 0.39605751633644104, + "r2": 0.5234577655792236, + "score": -0.5042286809765437 + }, + "val": { + "rmse": 0.5553348954063525, + "mae": 0.44016024470329285, + "r2": 0.4184489846229553, + "score": -0.5553348954063525 + }, + "test": { + "rmse": 0.550360417400049, + "mae": 0.44305530190467834, + "r2": 0.43557870388031006, + "score": -0.550360417400049 + } + }, + "time": "0:07:15.282501", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13, + 6, + 10, + 14, + 26, + 27, + 18, + 25, + 6, + 15, + 8, + 17, + 10, + 26 + ], + "head_selection_time": "0:00:00.438429" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0p5-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0p5-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0p5-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0p5-best-head-evaluation/0/report.json new file mode 100644 index 000000000..074f2755a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0p5-best-head-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5112224822345067, + "mae": 0.40589213371276855, + "r2": 0.5101465582847595, + "score": -0.5112224822345067 + }, + "val": { + "rmse": 0.5583881453279098, + "mae": 0.4439501166343689, + "r2": 0.412036657333374, + "score": -0.5583881453279098 + }, + "test": { + "rmse": 0.5557654209580059, + "mae": 0.4505227506160736, + "r2": 0.42443811893463135, + "score": -0.5557654209580059 + } + }, + "time": "0:07:16.646461", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023405" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0p5-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0p5-evaluation/0.toml new file mode 100644 index 000000000..f7d34ae59 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0p5-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.5 +rla_extra_paths_freeze_fraction = 0.5 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0p5-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0p5-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0p5-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0p5-evaluation/0/report.json new file mode 100644 index 000000000..831b044e8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0p5-evaluation/0/report.json @@ -0,0 +1,171 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5040057771555991, + "mae": 0.39488768577575684, + "r2": 0.5238790512084961, + "score": -0.5040057771555991 + }, + "val": { + "rmse": 0.5558749906446844, + "mae": 0.4398687481880188, + "r2": 0.4173172116279602, + "score": -0.5558749906446844 + }, + "test": { + "rmse": 0.5496162918403876, + "mae": 0.44109272956848145, + "r2": 0.4371039867401123, + "score": -0.5496162918403876 + } + }, + "time": "0:07:16.646461", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0p5-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0p5-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0p5-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0p5-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..9e9ee3780 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0p5-greedy-heads-evaluation/0/report.json @@ -0,0 +1,188 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5042286809765437, + "mae": 0.39605751633644104, + "r2": 0.5234577655792236, + "score": -0.5042286809765437 + }, + "val": { + "rmse": 0.5553348954063525, + "mae": 0.44016024470329285, + "r2": 0.4184489846229553, + "score": -0.5553348954063525 + }, + "test": { + "rmse": 0.550360417400049, + "mae": 0.44305530190467834, + "r2": 0.43557870388031006, + "score": -0.550360417400049 + } + }, + "time": "0:07:16.646461", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13, + 6, + 10, + 14, + 26, + 27, + 18, + 25, + 6, + 15, + 8, + 17, + 10, + 26 + ], + "head_selection_time": "0:00:00.435497" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0-best-head-evaluation/0/report.json new file mode 100644 index 000000000..ee6bb5730 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0-best-head-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5112224822345067, + "mae": 0.40589213371276855, + "r2": 0.5101465582847595, + "score": -0.5112224822345067 + }, + "val": { + "rmse": 0.5583881453279098, + "mae": 0.4439501166343689, + "r2": 0.412036657333374, + "score": -0.5583881453279098 + }, + "test": { + "rmse": 0.5557654209580059, + "mae": 0.4505227506160736, + "r2": 0.42443811893463135, + "score": -0.5557654209580059 + } + }, + "time": "0:07:16.567442", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023072" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0-evaluation/0.toml new file mode 100644 index 000000000..a6084c224 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.25 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0-evaluation/0/report.json new file mode 100644 index 000000000..c0683acdb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0-evaluation/0/report.json @@ -0,0 +1,171 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5040057771555991, + "mae": 0.39488768577575684, + "r2": 0.5238790512084961, + "score": -0.5040057771555991 + }, + "val": { + "rmse": 0.5558749906446844, + "mae": 0.4398687481880188, + "r2": 0.4173172116279602, + "score": -0.5558749906446844 + }, + "test": { + "rmse": 0.5496162918403876, + "mae": 0.44109272956848145, + "r2": 0.4371039867401123, + "score": -0.5496162918403876 + } + }, + "time": "0:07:16.567442", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..ae4079b97 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0-greedy-heads-evaluation/0/report.json @@ -0,0 +1,188 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5042286809765437, + "mae": 0.39605751633644104, + "r2": 0.5234577655792236, + "score": -0.5042286809765437 + }, + "val": { + "rmse": 0.5553348954063525, + "mae": 0.44016024470329285, + "r2": 0.4184489846229553, + "score": -0.5553348954063525 + }, + "test": { + "rmse": 0.550360417400049, + "mae": 0.44305530190467834, + "r2": 0.43557870388031006, + "score": -0.550360417400049 + } + }, + "time": "0:07:16.567442", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13, + 6, + 10, + 14, + 26, + 27, + 18, + 25, + 6, + 15, + 8, + 17, + 10, + 26 + ], + "head_selection_time": "0:00:00.436208" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0p5-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0p5-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0p5-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0p5-best-head-evaluation/0/report.json new file mode 100644 index 000000000..1e774b115 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0p5-best-head-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5112224822345067, + "mae": 0.40589213371276855, + "r2": 0.5101465582847595, + "score": -0.5112224822345067 + }, + "val": { + "rmse": 0.5583881453279098, + "mae": 0.4439501166343689, + "r2": 0.412036657333374, + "score": -0.5583881453279098 + }, + "test": { + "rmse": 0.5557654209580059, + "mae": 0.4505227506160736, + "r2": 0.42443811893463135, + "score": -0.5557654209580059 + } + }, + "time": "0:07:15.560400", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023828" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0p5-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0p5-evaluation/0.toml new file mode 100644 index 000000000..99e15b430 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0p5-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.25 +rla_extra_paths_freeze_fraction = 0.5 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0p5-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0p5-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0p5-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0p5-evaluation/0/report.json new file mode 100644 index 000000000..4089927f2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0p5-evaluation/0/report.json @@ -0,0 +1,171 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5040057771555991, + "mae": 0.39488768577575684, + "r2": 0.5238790512084961, + "score": -0.5040057771555991 + }, + "val": { + "rmse": 0.5558749906446844, + "mae": 0.4398687481880188, + "r2": 0.4173172116279602, + "score": -0.5558749906446844 + }, + "test": { + "rmse": 0.5496162918403876, + "mae": 0.44109272956848145, + "r2": 0.4371039867401123, + "score": -0.5496162918403876 + } + }, + "time": "0:07:15.560400", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0p5-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0p5-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0p5-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0p5-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..f96938ad2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0p5-greedy-heads-evaluation/0/report.json @@ -0,0 +1,188 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5042286809765437, + "mae": 0.39605751633644104, + "r2": 0.5234577655792236, + "score": -0.5042286809765437 + }, + "val": { + "rmse": 0.5553348954063525, + "mae": 0.44016024470329285, + "r2": 0.4184489846229553, + "score": -0.5553348954063525 + }, + "test": { + "rmse": 0.550360417400049, + "mae": 0.44305530190467834, + "r2": 0.43557870388031006, + "score": -0.550360417400049 + } + }, + "time": "0:07:15.560400", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13, + 6, + 10, + 14, + 26, + 27, + 18, + 25, + 6, + 15, + 8, + 17, + 10, + 26 + ], + "head_selection_time": "0:00:00.438707" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0-best-head-evaluation/0/report.json new file mode 100644 index 000000000..5081de80f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0-best-head-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5112224822345067, + "mae": 0.40589213371276855, + "r2": 0.5101465582847595, + "score": -0.5112224822345067 + }, + "val": { + "rmse": 0.5583881453279098, + "mae": 0.4439501166343689, + "r2": 0.412036657333374, + "score": -0.5583881453279098 + }, + "test": { + "rmse": 0.5557654209580059, + "mae": 0.4505227506160736, + "r2": 0.42443811893463135, + "score": -0.5557654209580059 + } + }, + "time": "0:07:16.936182", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023175" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0-evaluation/0.toml new file mode 100644 index 000000000..810456200 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.5 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0-evaluation/0/report.json new file mode 100644 index 000000000..232caab6a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0-evaluation/0/report.json @@ -0,0 +1,171 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5040057771555991, + "mae": 0.39488768577575684, + "r2": 0.5238790512084961, + "score": -0.5040057771555991 + }, + "val": { + "rmse": 0.5558749906446844, + "mae": 0.4398687481880188, + "r2": 0.4173172116279602, + "score": -0.5558749906446844 + }, + "test": { + "rmse": 0.5496162918403876, + "mae": 0.44109272956848145, + "r2": 0.4371039867401123, + "score": -0.5496162918403876 + } + }, + "time": "0:07:16.936182", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..704d7d218 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0-greedy-heads-evaluation/0/report.json @@ -0,0 +1,188 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5042286809765437, + "mae": 0.39605751633644104, + "r2": 0.5234577655792236, + "score": -0.5042286809765437 + }, + "val": { + "rmse": 0.5553348954063525, + "mae": 0.44016024470329285, + "r2": 0.4184489846229553, + "score": -0.5553348954063525 + }, + "test": { + "rmse": 0.550360417400049, + "mae": 0.44305530190467834, + "r2": 0.43557870388031006, + "score": -0.550360417400049 + } + }, + "time": "0:07:16.936182", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13, + 6, + 10, + 14, + 26, + 27, + 18, + 25, + 6, + 15, + 8, + 17, + 10, + 26 + ], + "head_selection_time": "0:00:00.447468" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0p5-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0p5-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0p5-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0p5-best-head-evaluation/0/report.json new file mode 100644 index 000000000..204ac32e0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0p5-best-head-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5112224822345067, + "mae": 0.40589213371276855, + "r2": 0.5101465582847595, + "score": -0.5112224822345067 + }, + "val": { + "rmse": 0.5583881453279098, + "mae": 0.4439501166343689, + "r2": 0.412036657333374, + "score": -0.5583881453279098 + }, + "test": { + "rmse": 0.5557654209580059, + "mae": 0.4505227506160736, + "r2": 0.42443811893463135, + "score": -0.5557654209580059 + } + }, + "time": "0:07:15.700493", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023475" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0p5-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0p5-evaluation/0.toml new file mode 100644 index 000000000..3ebdb92a5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0p5-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.5 +rla_extra_paths_freeze_fraction = 0.5 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0p5-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0p5-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0p5-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0p5-evaluation/0/report.json new file mode 100644 index 000000000..231e00372 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0p5-evaluation/0/report.json @@ -0,0 +1,171 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5040057771555991, + "mae": 0.39488768577575684, + "r2": 0.5238790512084961, + "score": -0.5040057771555991 + }, + "val": { + "rmse": 0.5558749906446844, + "mae": 0.4398687481880188, + "r2": 0.4173172116279602, + "score": -0.5558749906446844 + }, + "test": { + "rmse": 0.5496162918403876, + "mae": 0.44109272956848145, + "r2": 0.4371039867401123, + "score": -0.5496162918403876 + } + }, + "time": "0:07:15.700493", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0p5-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0p5-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0p5-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0p5-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..9fcc2a4bd --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0p5-greedy-heads-evaluation/0/report.json @@ -0,0 +1,188 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5042286809765437, + "mae": 0.39605751633644104, + "r2": 0.5234577655792236, + "score": -0.5042286809765437 + }, + "val": { + "rmse": 0.5553348954063525, + "mae": 0.44016024470329285, + "r2": 0.4184489846229553, + "score": -0.5553348954063525 + }, + "test": { + "rmse": 0.550360417400049, + "mae": 0.44305530190467834, + "r2": 0.43557870388031006, + "score": -0.550360417400049 + } + }, + "time": "0:07:15.700493", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13, + 6, + 10, + 14, + 26, + 27, + 18, + 25, + 6, + 15, + 8, + 17, + 10, + 26 + ], + "head_selection_time": "0:00:00.433892" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0-best-head-evaluation/0/report.json new file mode 100644 index 000000000..fcebf13d7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0-best-head-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5112224822345067, + "mae": 0.40589213371276855, + "r2": 0.5101465582847595, + "score": -0.5112224822345067 + }, + "val": { + "rmse": 0.5583881453279098, + "mae": 0.4439501166343689, + "r2": 0.412036657333374, + "score": -0.5583881453279098 + }, + "test": { + "rmse": 0.5557654209580059, + "mae": 0.4505227506160736, + "r2": 0.42443811893463135, + "score": -0.5557654209580059 + } + }, + "time": "0:07:15.962687", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023430" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0-evaluation/0.toml new file mode 100644 index 000000000..a1f4daf9d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.25 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0-evaluation/0/report.json new file mode 100644 index 000000000..65b7999ae --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0-evaluation/0/report.json @@ -0,0 +1,171 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5040057771555991, + "mae": 0.39488768577575684, + "r2": 0.5238790512084961, + "score": -0.5040057771555991 + }, + "val": { + "rmse": 0.5558749906446844, + "mae": 0.4398687481880188, + "r2": 0.4173172116279602, + "score": -0.5558749906446844 + }, + "test": { + "rmse": 0.5496162918403876, + "mae": 0.44109272956848145, + "r2": 0.4371039867401123, + "score": -0.5496162918403876 + } + }, + "time": "0:07:15.962687", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..46b9d1855 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0-greedy-heads-evaluation/0/report.json @@ -0,0 +1,188 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5042286809765437, + "mae": 0.39605751633644104, + "r2": 0.5234577655792236, + "score": -0.5042286809765437 + }, + "val": { + "rmse": 0.5553348954063525, + "mae": 0.44016024470329285, + "r2": 0.4184489846229553, + "score": -0.5553348954063525 + }, + "test": { + "rmse": 0.550360417400049, + "mae": 0.44305530190467834, + "r2": 0.43557870388031006, + "score": -0.550360417400049 + } + }, + "time": "0:07:15.962687", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13, + 6, + 10, + 14, + 26, + 27, + 18, + 25, + 6, + 15, + 8, + 17, + 10, + 26 + ], + "head_selection_time": "0:00:00.435584" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0p5-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0p5-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0p5-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0p5-best-head-evaluation/0/report.json new file mode 100644 index 000000000..8ba411d31 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0p5-best-head-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5112224822345067, + "mae": 0.40589213371276855, + "r2": 0.5101465582847595, + "score": -0.5112224822345067 + }, + "val": { + "rmse": 0.5583881453279098, + "mae": 0.4439501166343689, + "r2": 0.412036657333374, + "score": -0.5583881453279098 + }, + "test": { + "rmse": 0.5557654209580059, + "mae": 0.4505227506160736, + "r2": 0.42443811893463135, + "score": -0.5557654209580059 + } + }, + "time": "0:07:17.779232", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023298" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0p5-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0p5-evaluation/0.toml new file mode 100644 index 000000000..b422ea477 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0p5-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.25 +rla_extra_paths_freeze_fraction = 0.5 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0p5-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0p5-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0p5-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0p5-evaluation/0/report.json new file mode 100644 index 000000000..c0ce5f682 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0p5-evaluation/0/report.json @@ -0,0 +1,171 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5040057771555991, + "mae": 0.39488768577575684, + "r2": 0.5238790512084961, + "score": -0.5040057771555991 + }, + "val": { + "rmse": 0.5558749906446844, + "mae": 0.4398687481880188, + "r2": 0.4173172116279602, + "score": -0.5558749906446844 + }, + "test": { + "rmse": 0.5496162918403876, + "mae": 0.44109272956848145, + "r2": 0.4371039867401123, + "score": -0.5496162918403876 + } + }, + "time": "0:07:17.779232", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0p5-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0p5-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0p5-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0p5-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..f81343fae --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0p5-greedy-heads-evaluation/0/report.json @@ -0,0 +1,188 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5042286809765437, + "mae": 0.39605751633644104, + "r2": 0.5234577655792236, + "score": -0.5042286809765437 + }, + "val": { + "rmse": 0.5553348954063525, + "mae": 0.44016024470329285, + "r2": 0.4184489846229553, + "score": -0.5553348954063525 + }, + "test": { + "rmse": 0.550360417400049, + "mae": 0.44305530190467834, + "r2": 0.43557870388031006, + "score": -0.550360417400049 + } + }, + "time": "0:07:17.779232", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13, + 6, + 10, + 14, + 26, + 27, + 18, + 25, + 6, + 15, + 8, + 17, + 10, + 26 + ], + "head_selection_time": "0:00:00.465491" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0-best-head-evaluation/0/report.json new file mode 100644 index 000000000..ba9b7c54f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0-best-head-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5112224822345067, + "mae": 0.40589213371276855, + "r2": 0.5101465582847595, + "score": -0.5112224822345067 + }, + "val": { + "rmse": 0.5583881453279098, + "mae": 0.4439501166343689, + "r2": 0.412036657333374, + "score": -0.5583881453279098 + }, + "test": { + "rmse": 0.5557654209580059, + "mae": 0.4505227506160736, + "r2": 0.42443811893463135, + "score": -0.5557654209580059 + } + }, + "time": "0:07:16.838062", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023005" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0-evaluation/0.toml new file mode 100644 index 000000000..9ae3fb7b7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.5 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0-evaluation/0/report.json new file mode 100644 index 000000000..ed9cd7b72 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0-evaluation/0/report.json @@ -0,0 +1,171 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5040057771555991, + "mae": 0.39488768577575684, + "r2": 0.5238790512084961, + "score": -0.5040057771555991 + }, + "val": { + "rmse": 0.5558749906446844, + "mae": 0.4398687481880188, + "r2": 0.4173172116279602, + "score": -0.5558749906446844 + }, + "test": { + "rmse": 0.5496162918403876, + "mae": 0.44109272956848145, + "r2": 0.4371039867401123, + "score": -0.5496162918403876 + } + }, + "time": "0:07:16.838062", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..463950df0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0-greedy-heads-evaluation/0/report.json @@ -0,0 +1,188 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5042286809765437, + "mae": 0.39605751633644104, + "r2": 0.5234577655792236, + "score": -0.5042286809765437 + }, + "val": { + "rmse": 0.5553348954063525, + "mae": 0.44016024470329285, + "r2": 0.4184489846229553, + "score": -0.5553348954063525 + }, + "test": { + "rmse": 0.550360417400049, + "mae": 0.44305530190467834, + "r2": 0.43557870388031006, + "score": -0.550360417400049 + } + }, + "time": "0:07:16.838062", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13, + 6, + 10, + 14, + 26, + 27, + 18, + 25, + 6, + 15, + 8, + 17, + 10, + 26 + ], + "head_selection_time": "0:00:00.421543" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0p5-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0p5-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0p5-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0p5-best-head-evaluation/0/report.json new file mode 100644 index 000000000..3b2092ff4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0p5-best-head-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5112224822345067, + "mae": 0.40589213371276855, + "r2": 0.5101465582847595, + "score": -0.5112224822345067 + }, + "val": { + "rmse": 0.5583881453279098, + "mae": 0.4439501166343689, + "r2": 0.412036657333374, + "score": -0.5583881453279098 + }, + "test": { + "rmse": 0.5557654209580059, + "mae": 0.4505227506160736, + "r2": 0.42443811893463135, + "score": -0.5557654209580059 + } + }, + "time": "0:07:17.037029", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.024109" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0p5-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0p5-evaluation/0.toml new file mode 100644 index 000000000..dad6cd011 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0p5-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.5 +rla_extra_paths_freeze_fraction = 0.5 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0p5-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0p5-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0p5-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0p5-evaluation/0/report.json new file mode 100644 index 000000000..243854e1b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0p5-evaluation/0/report.json @@ -0,0 +1,171 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5040057771555991, + "mae": 0.39488768577575684, + "r2": 0.5238790512084961, + "score": -0.5040057771555991 + }, + "val": { + "rmse": 0.5558749906446844, + "mae": 0.4398687481880188, + "r2": 0.4173172116279602, + "score": -0.5558749906446844 + }, + "test": { + "rmse": 0.5496162918403876, + "mae": 0.44109272956848145, + "r2": 0.4371039867401123, + "score": -0.5496162918403876 + } + }, + "time": "0:07:17.037029", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0p5-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0p5-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0p5-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0p5-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..b1095bbb7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0p5-greedy-heads-evaluation/0/report.json @@ -0,0 +1,188 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5042286809765437, + "mae": 0.39605751633644104, + "r2": 0.5234577655792236, + "score": -0.5042286809765437 + }, + "val": { + "rmse": 0.5553348954063525, + "mae": 0.44016024470329285, + "r2": 0.4184489846229553, + "score": -0.5553348954063525 + }, + "test": { + "rmse": 0.550360417400049, + "mae": 0.44305530190467834, + "r2": 0.43557870388031006, + "score": -0.550360417400049 + } + }, + "time": "0:07:17.037029", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13, + 6, + 10, + 14, + 26, + 27, + 18, + 25, + 6, + 15, + 8, + 17, + 10, + 26 + ], + "head_selection_time": "0:00:00.446950" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0-best-head-evaluation/0/report.json new file mode 100644 index 000000000..17f4e1dc6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0-best-head-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5159634144292107, + "mae": 0.410226047039032, + "r2": 0.5010189414024353, + "score": -0.5159634144292107 + }, + "val": { + "rmse": 0.5593939165294649, + "mae": 0.4475388526916504, + "r2": 0.40991663932800293, + "score": -0.5593939165294649 + }, + "test": { + "rmse": 0.5608508784321002, + "mae": 0.45661959052085876, + "r2": 0.41385674476623535, + "score": -0.5608508784321002 + } + }, + "time": "0:10:51.385954", + "diversity": { + "train": { + "mean_centered_corr": -0.019908307120203972, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07479876279830933 + }, + "val": { + "mean_centered_corr": -0.02006814442574978, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07624257355928421 + }, + "test": { + "mean_centered_corr": -0.019939016550779343, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07578995823860168 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.023221" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0-evaluation/0.toml new file mode 100644 index 000000000..71616aded --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.25 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0-evaluation/0/report.json new file mode 100644 index 000000000..a0394b28b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5088939009594995, + "mae": 0.40171560645103455, + "r2": 0.5145988464355469, + "score": -0.5088939009594995 + }, + "val": { + "rmse": 0.5558022325623276, + "mae": 0.44188886880874634, + "r2": 0.4174697995185852, + "score": -0.5558022325623276 + }, + "test": { + "rmse": 0.5512480144508224, + "mae": 0.4457772672176361, + "r2": 0.4337567090988159, + "score": -0.5512480144508224 + } + }, + "time": "0:10:51.385954", + "diversity": { + "train": { + "mean_centered_corr": -0.019908307120203972, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07479876279830933 + }, + "val": { + "mean_centered_corr": -0.02006814442574978, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07624257355928421 + }, + "test": { + "mean_centered_corr": -0.019939016550779343, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07578995823860168 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..1e2adcd92 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0-greedy-heads-evaluation/0/report.json @@ -0,0 +1,191 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5086880691661995, + "mae": 0.4017886221408844, + "r2": 0.5149915218353271, + "score": -0.5086880691661995 + }, + "val": { + "rmse": 0.5554104781133956, + "mae": 0.44194847345352173, + "r2": 0.4182906746864319, + "score": -0.5554104781133956 + }, + "test": { + "rmse": 0.5525622419967886, + "mae": 0.4474184215068817, + "r2": 0.4310535788536072, + "score": -0.5525622419967886 + } + }, + "time": "0:10:51.385954", + "diversity": { + "train": { + "mean_centered_corr": -0.019908307120203972, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07479876279830933 + }, + "val": { + "mean_centered_corr": -0.02006814442574978, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07624257355928421 + }, + "test": { + "mean_centered_corr": -0.019939016550779343, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07578995823860168 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 4, + 13, + 18, + 11, + 15, + 6, + 21, + 19, + 29, + 31, + 16, + 14, + 22 + ], + "head_selection_time": "0:00:00.387723" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0p5-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0p5-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0p5-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0p5-best-head-evaluation/0/report.json new file mode 100644 index 000000000..88166125f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0p5-best-head-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5159634144292107, + "mae": 0.410226047039032, + "r2": 0.5010189414024353, + "score": -0.5159634144292107 + }, + "val": { + "rmse": 0.5593939165294649, + "mae": 0.4475388526916504, + "r2": 0.40991663932800293, + "score": -0.5593939165294649 + }, + "test": { + "rmse": 0.5608508784321002, + "mae": 0.45661959052085876, + "r2": 0.41385674476623535, + "score": -0.5608508784321002 + } + }, + "time": "0:10:52.470388", + "diversity": { + "train": { + "mean_centered_corr": -0.019908307120203972, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07479876279830933 + }, + "val": { + "mean_centered_corr": -0.02006814442574978, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07624257355928421 + }, + "test": { + "mean_centered_corr": -0.019939016550779343, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07578995823860168 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.023417" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0p5-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0p5-evaluation/0.toml new file mode 100644 index 000000000..fd11e9380 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0p5-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.25 +rla_extra_paths_freeze_fraction = 0.5 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0p5-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0p5-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0p5-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0p5-evaluation/0/report.json new file mode 100644 index 000000000..aa06b0f20 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0p5-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5088939009594995, + "mae": 0.40171560645103455, + "r2": 0.5145988464355469, + "score": -0.5088939009594995 + }, + "val": { + "rmse": 0.5558022325623276, + "mae": 0.44188886880874634, + "r2": 0.4174697995185852, + "score": -0.5558022325623276 + }, + "test": { + "rmse": 0.5512480144508224, + "mae": 0.4457772672176361, + "r2": 0.4337567090988159, + "score": -0.5512480144508224 + } + }, + "time": "0:10:52.470388", + "diversity": { + "train": { + "mean_centered_corr": -0.019908307120203972, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07479876279830933 + }, + "val": { + "mean_centered_corr": -0.02006814442574978, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07624257355928421 + }, + "test": { + "mean_centered_corr": -0.019939016550779343, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07578995823860168 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0p5-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0p5-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0p5-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0p5-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..41ea18a38 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0p5-greedy-heads-evaluation/0/report.json @@ -0,0 +1,191 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5086880691661995, + "mae": 0.4017886221408844, + "r2": 0.5149915218353271, + "score": -0.5086880691661995 + }, + "val": { + "rmse": 0.5554104781133956, + "mae": 0.44194847345352173, + "r2": 0.4182906746864319, + "score": -0.5554104781133956 + }, + "test": { + "rmse": 0.5525622419967886, + "mae": 0.4474184215068817, + "r2": 0.4310535788536072, + "score": -0.5525622419967886 + } + }, + "time": "0:10:52.470388", + "diversity": { + "train": { + "mean_centered_corr": -0.019908307120203972, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07479876279830933 + }, + "val": { + "mean_centered_corr": -0.02006814442574978, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07624257355928421 + }, + "test": { + "mean_centered_corr": -0.019939016550779343, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07578995823860168 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 4, + 13, + 18, + 11, + 15, + 6, + 21, + 19, + 29, + 31, + 16, + 14, + 22 + ], + "head_selection_time": "0:00:00.393613" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0-best-head-evaluation/0/report.json new file mode 100644 index 000000000..334f39a2a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0-best-head-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 3276, + "metrics": { + "train": { + "rmse": 0.502955286059696, + "mae": 0.3946622908115387, + "r2": 0.5258617401123047, + "score": -0.502955286059696 + }, + "val": { + "rmse": 0.5596610856119687, + "mae": 0.44277745485305786, + "r2": 0.4093528389930725, + "score": -0.5596610856119687 + }, + "test": { + "rmse": 0.5563749515907954, + "mae": 0.44843393564224243, + "r2": 0.4231749176979065, + "score": -0.5563749515907954 + } + }, + "time": "0:12:43.917677", + "diversity": { + "train": { + "mean_centered_corr": -0.020183589309453964, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0822177454829216 + }, + "val": { + "mean_centered_corr": -0.020031888037919998, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08821859210729599 + }, + "test": { + "mean_centered_corr": -0.01992768421769142, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08592649549245834 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023354" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0-evaluation/0.toml new file mode 100644 index 000000000..6079ea513 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.5 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0-evaluation/0/report.json new file mode 100644 index 000000000..9550fb3d3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 3276, + "metrics": { + "train": { + "rmse": 0.4956110911684622, + "mae": 0.3902001976966858, + "r2": 0.5396074056625366, + "score": -0.4956110911684622 + }, + "val": { + "rmse": 0.5558360123700841, + "mae": 0.43872880935668945, + "r2": 0.41739898920059204, + "score": -0.5558360123700841 + }, + "test": { + "rmse": 0.5491424149361828, + "mae": 0.4410218894481659, + "r2": 0.43807417154312134, + "score": -0.5491424149361828 + } + }, + "time": "0:12:43.917677", + "diversity": { + "train": { + "mean_centered_corr": -0.020183589309453964, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0822177454829216 + }, + "val": { + "mean_centered_corr": -0.020031888037919998, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08821859210729599 + }, + "test": { + "mean_centered_corr": -0.01992768421769142, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08592649549245834 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..77d313c47 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0-greedy-heads-evaluation/0/report.json @@ -0,0 +1,194 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 3276, + "metrics": { + "train": { + "rmse": 0.4960975196274642, + "mae": 0.3916766345500946, + "r2": 0.5387032628059387, + "score": -0.4960975196274642 + }, + "val": { + "rmse": 0.5552555453020565, + "mae": 0.4401964545249939, + "r2": 0.4186151623725891, + "score": -0.5552555453020565 + }, + "test": { + "rmse": 0.551575730248866, + "mae": 0.44500499963760376, + "r2": 0.43308329582214355, + "score": -0.551575730248866 + } + }, + "time": "0:12:43.917677", + "diversity": { + "train": { + "mean_centered_corr": -0.020183589309453964, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0822177454829216 + }, + "val": { + "mean_centered_corr": -0.020031888037919998, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08821859210729599 + }, + "test": { + "mean_centered_corr": -0.01992768421769142, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08592649549245834 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13, + 10, + 2, + 20, + 16, + 6, + 26, + 17, + 21, + 15, + 1, + 10, + 22, + 2, + 30, + 31 + ], + "head_selection_time": "0:00:00.511154" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0p5-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0p5-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0p5-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0p5-best-head-evaluation/0/report.json new file mode 100644 index 000000000..249e8ed6e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0p5-best-head-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 3276, + "metrics": { + "train": { + "rmse": 0.502955286059696, + "mae": 0.3946622908115387, + "r2": 0.5258617401123047, + "score": -0.502955286059696 + }, + "val": { + "rmse": 0.5596610856119687, + "mae": 0.44277745485305786, + "r2": 0.4093528389930725, + "score": -0.5596610856119687 + }, + "test": { + "rmse": 0.5563749515907954, + "mae": 0.44843393564224243, + "r2": 0.4231749176979065, + "score": -0.5563749515907954 + } + }, + "time": "0:12:43.404674", + "diversity": { + "train": { + "mean_centered_corr": -0.020183589309453964, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0822177454829216 + }, + "val": { + "mean_centered_corr": -0.020031888037919998, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08821859210729599 + }, + "test": { + "mean_centered_corr": -0.01992768421769142, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08592649549245834 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023207" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0p5-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0p5-evaluation/0.toml new file mode 100644 index 000000000..d0652b358 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0p5-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.5 +rla_extra_paths_freeze_fraction = 0.5 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0p5-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0p5-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0p5-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0p5-evaluation/0/report.json new file mode 100644 index 000000000..b98eef892 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0p5-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 3276, + "metrics": { + "train": { + "rmse": 0.4956110911684622, + "mae": 0.3902001976966858, + "r2": 0.5396074056625366, + "score": -0.4956110911684622 + }, + "val": { + "rmse": 0.5558360123700841, + "mae": 0.43872880935668945, + "r2": 0.41739898920059204, + "score": -0.5558360123700841 + }, + "test": { + "rmse": 0.5491424149361828, + "mae": 0.4410218894481659, + "r2": 0.43807417154312134, + "score": -0.5491424149361828 + } + }, + "time": "0:12:43.404674", + "diversity": { + "train": { + "mean_centered_corr": -0.020183589309453964, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0822177454829216 + }, + "val": { + "mean_centered_corr": -0.020031888037919998, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08821859210729599 + }, + "test": { + "mean_centered_corr": -0.01992768421769142, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08592649549245834 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0p5-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0p5-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0p5-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0p5-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..8d7085876 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0p5-greedy-heads-evaluation/0/report.json @@ -0,0 +1,194 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 3276, + "metrics": { + "train": { + "rmse": 0.4960975196274642, + "mae": 0.3916766345500946, + "r2": 0.5387032628059387, + "score": -0.4960975196274642 + }, + "val": { + "rmse": 0.5552555453020565, + "mae": 0.4401964545249939, + "r2": 0.4186151623725891, + "score": -0.5552555453020565 + }, + "test": { + "rmse": 0.551575730248866, + "mae": 0.44500499963760376, + "r2": 0.43308329582214355, + "score": -0.551575730248866 + } + }, + "time": "0:12:43.404674", + "diversity": { + "train": { + "mean_centered_corr": -0.020183589309453964, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0822177454829216 + }, + "val": { + "mean_centered_corr": -0.020031888037919998, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08821859210729599 + }, + "test": { + "mean_centered_corr": -0.01992768421769142, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08592649549245834 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13, + 10, + 2, + 20, + 16, + 6, + 26, + 17, + 21, + 15, + 1, + 10, + 22, + 2, + 30, + 31 + ], + "head_selection_time": "0:00:00.511173" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0-best-head-evaluation/0/report.json new file mode 100644 index 000000000..1ea412fe3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0-best-head-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5127050489681885, + "mae": 0.40456077456474304, + "r2": 0.5073012113571167, + "score": -0.5127050489681885 + }, + "val": { + "rmse": 0.5586283212914263, + "mae": 0.4427802264690399, + "r2": 0.41153067350387573, + "score": -0.5586283212914263 + }, + "test": { + "rmse": 0.5508851568810451, + "mae": 0.4434031546115875, + "r2": 0.4345019459724426, + "score": -0.5508851568810451 + } + }, + "time": "0:10:54.308917", + "diversity": { + "train": { + "mean_centered_corr": -0.020132694393396378, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07372817397117615 + }, + "val": { + "mean_centered_corr": -0.02042890153825283, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07545945793390274 + }, + "test": { + "mean_centered_corr": -0.020171180367469788, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07420117408037186 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 10 + ], + "head_selection_time": "0:00:00.023488" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0-evaluation/0.toml new file mode 100644 index 000000000..f7b19054f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.25 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0-evaluation/0/report.json new file mode 100644 index 000000000..151ef7a99 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5087820041628177, + "mae": 0.4012722671031952, + "r2": 0.5148123502731323, + "score": -0.5087820041628177 + }, + "val": { + "rmse": 0.5555827210327928, + "mae": 0.44155657291412354, + "r2": 0.41792982816696167, + "score": -0.5555827210327928 + }, + "test": { + "rmse": 0.5515477953887775, + "mae": 0.44571202993392944, + "r2": 0.43314069509506226, + "score": -0.5515477953887775 + } + }, + "time": "0:10:54.308917", + "diversity": { + "train": { + "mean_centered_corr": -0.020132694393396378, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07372817397117615 + }, + "val": { + "mean_centered_corr": -0.02042890153825283, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07545945793390274 + }, + "test": { + "mean_centered_corr": -0.020171180367469788, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07420117408037186 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..af1922030 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0-greedy-heads-evaluation/0/report.json @@ -0,0 +1,188 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5091947669637598, + "mae": 0.40187934041023254, + "r2": 0.5140247344970703, + "score": -0.5091947669637598 + }, + "val": { + "rmse": 0.5550703688421238, + "mae": 0.44088995456695557, + "r2": 0.41900283098220825, + "score": -0.5550703688421238 + }, + "test": { + "rmse": 0.5524078860674355, + "mae": 0.44680875539779663, + "r2": 0.4313713312149048, + "score": -0.5524078860674355 + } + }, + "time": "0:10:54.308917", + "diversity": { + "train": { + "mean_centered_corr": -0.020132694393396378, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07372817397117615 + }, + "val": { + "mean_centered_corr": -0.02042890153825283, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07545945793390274 + }, + "test": { + "mean_centered_corr": -0.020171180367469788, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07420117408037186 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 10, + 14, + 15, + 25, + 6, + 18, + 26, + 21, + 13, + 4 + ], + "head_selection_time": "0:00:00.287558" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0p5-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0p5-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0p5-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0p5-best-head-evaluation/0/report.json new file mode 100644 index 000000000..102ae55df --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0p5-best-head-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5159634144292107, + "mae": 0.410226047039032, + "r2": 0.5010189414024353, + "score": -0.5159634144292107 + }, + "val": { + "rmse": 0.5593939165294649, + "mae": 0.4475388526916504, + "r2": 0.40991663932800293, + "score": -0.5593939165294649 + }, + "test": { + "rmse": 0.5608508784321002, + "mae": 0.45661959052085876, + "r2": 0.41385674476623535, + "score": -0.5608508784321002 + } + }, + "time": "0:10:53.683873", + "diversity": { + "train": { + "mean_centered_corr": -0.019908307120203972, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07479876279830933 + }, + "val": { + "mean_centered_corr": -0.02006814442574978, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07624257355928421 + }, + "test": { + "mean_centered_corr": -0.019939016550779343, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07578995823860168 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.023809" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0p5-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0p5-evaluation/0.toml new file mode 100644 index 000000000..3c50600be --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0p5-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.25 +rla_extra_paths_freeze_fraction = 0.5 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0p5-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0p5-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0p5-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0p5-evaluation/0/report.json new file mode 100644 index 000000000..13022f6f8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0p5-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5088939009594995, + "mae": 0.40171560645103455, + "r2": 0.5145988464355469, + "score": -0.5088939009594995 + }, + "val": { + "rmse": 0.5558022325623276, + "mae": 0.44188886880874634, + "r2": 0.4174697995185852, + "score": -0.5558022325623276 + }, + "test": { + "rmse": 0.5512480144508224, + "mae": 0.4457772672176361, + "r2": 0.4337567090988159, + "score": -0.5512480144508224 + } + }, + "time": "0:10:53.683873", + "diversity": { + "train": { + "mean_centered_corr": -0.019908307120203972, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07479876279830933 + }, + "val": { + "mean_centered_corr": -0.02006814442574978, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07624257355928421 + }, + "test": { + "mean_centered_corr": -0.019939016550779343, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07578995823860168 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0p5-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0p5-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0p5-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0p5-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..42bfb1dbd --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0p5-greedy-heads-evaluation/0/report.json @@ -0,0 +1,191 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5086880691661995, + "mae": 0.4017886221408844, + "r2": 0.5149915218353271, + "score": -0.5086880691661995 + }, + "val": { + "rmse": 0.5554104781133956, + "mae": 0.44194847345352173, + "r2": 0.4182906746864319, + "score": -0.5554104781133956 + }, + "test": { + "rmse": 0.5525622419967886, + "mae": 0.4474184215068817, + "r2": 0.4310535788536072, + "score": -0.5525622419967886 + } + }, + "time": "0:10:53.683873", + "diversity": { + "train": { + "mean_centered_corr": -0.019908307120203972, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07479876279830933 + }, + "val": { + "mean_centered_corr": -0.02006814442574978, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07624257355928421 + }, + "test": { + "mean_centered_corr": -0.019939016550779343, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07578995823860168 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 4, + 13, + 18, + 11, + 15, + 6, + 21, + 19, + 29, + 31, + 16, + 14, + 22 + ], + "head_selection_time": "0:00:00.412208" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0-best-head-evaluation/0/report.json new file mode 100644 index 000000000..b80356461 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0-best-head-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5123672452974037, + "mae": 0.4070248603820801, + "r2": 0.5079503059387207, + "score": -0.5123672452974037 + }, + "val": { + "rmse": 0.5584356978034605, + "mae": 0.4474104344844818, + "r2": 0.4119364619255066, + "score": -0.5584356978034605 + }, + "test": { + "rmse": 0.5561328378795123, + "mae": 0.4517236351966858, + "r2": 0.42367684841156006, + "score": -0.5561328378795123 + } + }, + "time": "0:10:53.886867", + "diversity": { + "train": { + "mean_centered_corr": -0.020328762009739876, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07261504977941513 + }, + "val": { + "mean_centered_corr": -0.020213842391967773, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07467607408761978 + }, + "test": { + "mean_centered_corr": -0.0204370878636837, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07374871522188187 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 10 + ], + "head_selection_time": "0:00:00.023041" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0-evaluation/0.toml new file mode 100644 index 000000000..7a4a15b7a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.5 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0-evaluation/0/report.json new file mode 100644 index 000000000..0f7eca095 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5069226317597442, + "mae": 0.3994384706020355, + "r2": 0.5183521509170532, + "score": -0.5069226317597442 + }, + "val": { + "rmse": 0.5557252820018314, + "mae": 0.44164222478866577, + "r2": 0.4176310896873474, + "score": -0.5557252820018314 + }, + "test": { + "rmse": 0.5515971532207388, + "mae": 0.44512292742729187, + "r2": 0.43303918838500977, + "score": -0.5515971532207388 + } + }, + "time": "0:10:53.886867", + "diversity": { + "train": { + "mean_centered_corr": -0.020328762009739876, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07261504977941513 + }, + "val": { + "mean_centered_corr": -0.020213842391967773, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07467607408761978 + }, + "test": { + "mean_centered_corr": -0.0204370878636837, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07374871522188187 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..dd358b3a9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0-greedy-heads-evaluation/0/report.json @@ -0,0 +1,185 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5084577129451613, + "mae": 0.4028453230857849, + "r2": 0.5154306292533875, + "score": -0.5084577129451613 + }, + "val": { + "rmse": 0.555272076390732, + "mae": 0.44189491868019104, + "r2": 0.4185805320739746, + "score": -0.555272076390732 + }, + "test": { + "rmse": 0.5536843483543759, + "mae": 0.4483037292957306, + "r2": 0.4287404417991638, + "score": -0.5536843483543759 + } + }, + "time": "0:10:53.886867", + "diversity": { + "train": { + "mean_centered_corr": -0.020328762009739876, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07261504977941513 + }, + "val": { + "mean_centered_corr": -0.020213842391967773, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07467607408761978 + }, + "test": { + "mean_centered_corr": -0.0204370878636837, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07374871522188187 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 10, + 6, + 4, + 13, + 8, + 25, + 18 + ], + "head_selection_time": "0:00:00.203331" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0p5-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0p5-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0p5-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0p5-best-head-evaluation/0/report.json new file mode 100644 index 000000000..f6a3eac71 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0p5-best-head-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 3276, + "metrics": { + "train": { + "rmse": 0.502955286059696, + "mae": 0.3946622908115387, + "r2": 0.5258617401123047, + "score": -0.502955286059696 + }, + "val": { + "rmse": 0.5596610856119687, + "mae": 0.44277745485305786, + "r2": 0.4093528389930725, + "score": -0.5596610856119687 + }, + "test": { + "rmse": 0.5563749515907954, + "mae": 0.44843393564224243, + "r2": 0.4231749176979065, + "score": -0.5563749515907954 + } + }, + "time": "0:12:42.623804", + "diversity": { + "train": { + "mean_centered_corr": -0.020183589309453964, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0822177454829216 + }, + "val": { + "mean_centered_corr": -0.020031888037919998, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08821859210729599 + }, + "test": { + "mean_centered_corr": -0.01992768421769142, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08592649549245834 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023268" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0p5-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0p5-evaluation/0.toml new file mode 100644 index 000000000..e79913e2b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0p5-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.5 +rla_extra_paths_freeze_fraction = 0.5 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0p5-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0p5-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0p5-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0p5-evaluation/0/report.json new file mode 100644 index 000000000..f3b4f49f6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0p5-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 3276, + "metrics": { + "train": { + "rmse": 0.4956110911684622, + "mae": 0.3902001976966858, + "r2": 0.5396074056625366, + "score": -0.4956110911684622 + }, + "val": { + "rmse": 0.5558360123700841, + "mae": 0.43872880935668945, + "r2": 0.41739898920059204, + "score": -0.5558360123700841 + }, + "test": { + "rmse": 0.5491424149361828, + "mae": 0.4410218894481659, + "r2": 0.43807417154312134, + "score": -0.5491424149361828 + } + }, + "time": "0:12:42.623804", + "diversity": { + "train": { + "mean_centered_corr": -0.020183589309453964, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0822177454829216 + }, + "val": { + "mean_centered_corr": -0.020031888037919998, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08821859210729599 + }, + "test": { + "mean_centered_corr": -0.01992768421769142, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08592649549245834 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0p5-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0p5-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0p5-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0p5-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..7de17cfbc --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0p5-greedy-heads-evaluation/0/report.json @@ -0,0 +1,194 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 3276, + "metrics": { + "train": { + "rmse": 0.4960975196274642, + "mae": 0.3916766345500946, + "r2": 0.5387032628059387, + "score": -0.4960975196274642 + }, + "val": { + "rmse": 0.5552555453020565, + "mae": 0.4401964545249939, + "r2": 0.4186151623725891, + "score": -0.5552555453020565 + }, + "test": { + "rmse": 0.551575730248866, + "mae": 0.44500499963760376, + "r2": 0.43308329582214355, + "score": -0.551575730248866 + } + }, + "time": "0:12:42.623804", + "diversity": { + "train": { + "mean_centered_corr": -0.020183589309453964, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0822177454829216 + }, + "val": { + "mean_centered_corr": -0.020031888037919998, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08821859210729599 + }, + "test": { + "mean_centered_corr": -0.01992768421769142, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08592649549245834 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13, + 10, + 2, + 20, + 16, + 6, + 26, + 17, + 21, + 15, + 1, + 10, + 22, + 2, + 30, + 31 + ], + "head_selection_time": "0:00:00.507256" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0-best-head-evaluation/0/report.json new file mode 100644 index 000000000..f552fa0c2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0-best-head-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 3276, + "metrics": { + "train": { + "rmse": 0.5090104864160705, + "mae": 0.4024014472961426, + "r2": 0.5143764615058899, + "score": -0.5090104864160705 + }, + "val": { + "rmse": 0.5603404179951016, + "mae": 0.4449836015701294, + "r2": 0.4079180955886841, + "score": -0.5603404179951016 + }, + "test": { + "rmse": 0.5552489703004402, + "mae": 0.4475087821483612, + "r2": 0.4255073070526123, + "score": -0.5552489703004402 + } + }, + "time": "0:12:41.926992", + "diversity": { + "train": { + "mean_centered_corr": -0.020042935386300087, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0816296711564064 + }, + "val": { + "mean_centered_corr": -0.02009006403386593, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0869147852063179 + }, + "test": { + "mean_centered_corr": -0.020102225244045258, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08457230031490326 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 1 + ], + "head_selection_time": "0:00:00.023517" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0-evaluation/0.toml new file mode 100644 index 000000000..7d30e317b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.25 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0-evaluation/0/report.json new file mode 100644 index 000000000..fd1873d88 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 3276, + "metrics": { + "train": { + "rmse": 0.49865310275031377, + "mae": 0.3922213613986969, + "r2": 0.5339384078979492, + "score": -0.49865310275031377 + }, + "val": { + "rmse": 0.5560690371152802, + "mae": 0.438926637172699, + "r2": 0.4169103503227234, + "score": -0.5560690371152802 + }, + "test": { + "rmse": 0.5483022050176959, + "mae": 0.4402907192707062, + "r2": 0.4397923946380615, + "score": -0.5483022050176959 + } + }, + "time": "0:12:41.926992", + "diversity": { + "train": { + "mean_centered_corr": -0.020042935386300087, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0816296711564064 + }, + "val": { + "mean_centered_corr": -0.02009006403386593, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0869147852063179 + }, + "test": { + "mean_centered_corr": -0.020102225244045258, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08457230031490326 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..c50e32901 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0-greedy-heads-evaluation/0/report.json @@ -0,0 +1,190 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 3276, + "metrics": { + "train": { + "rmse": 0.5000589753179171, + "mae": 0.3948548436164856, + "r2": 0.5313067436218262, + "score": -0.5000589753179171 + }, + "val": { + "rmse": 0.5554431013363782, + "mae": 0.4404180943965912, + "r2": 0.4182223081588745, + "score": -0.5554431013363782 + }, + "test": { + "rmse": 0.5505825252366326, + "mae": 0.44418030977249146, + "r2": 0.435123085975647, + "score": -0.5505825252366326 + } + }, + "time": "0:12:41.926992", + "diversity": { + "train": { + "mean_centered_corr": -0.020042935386300087, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0816296711564064 + }, + "val": { + "mean_centered_corr": -0.02009006403386593, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0869147852063179 + }, + "test": { + "mean_centered_corr": -0.020102225244045258, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08457230031490326 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 1, + 13, + 22, + 18, + 17, + 20, + 11, + 12, + 29, + 6, + 14, + 26 + ], + "head_selection_time": "0:00:00.374219" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0p5-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0p5-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0p5-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0p5-best-head-evaluation/0/report.json new file mode 100644 index 000000000..96a17f586 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0p5-best-head-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5159634144292107, + "mae": 0.410226047039032, + "r2": 0.5010189414024353, + "score": -0.5159634144292107 + }, + "val": { + "rmse": 0.5593939165294649, + "mae": 0.4475388526916504, + "r2": 0.40991663932800293, + "score": -0.5593939165294649 + }, + "test": { + "rmse": 0.5608508784321002, + "mae": 0.45661959052085876, + "r2": 0.41385674476623535, + "score": -0.5608508784321002 + } + }, + "time": "0:10:54.207858", + "diversity": { + "train": { + "mean_centered_corr": -0.019908307120203972, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07479876279830933 + }, + "val": { + "mean_centered_corr": -0.02006814442574978, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07624257355928421 + }, + "test": { + "mean_centered_corr": -0.019939016550779343, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07578995823860168 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.023310" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0p5-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0p5-evaluation/0.toml new file mode 100644 index 000000000..210a8f378 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0p5-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.25 +rla_extra_paths_freeze_fraction = 0.5 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0p5-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0p5-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0p5-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0p5-evaluation/0/report.json new file mode 100644 index 000000000..1fee1c9c3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0p5-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5088939009594995, + "mae": 0.40171560645103455, + "r2": 0.5145988464355469, + "score": -0.5088939009594995 + }, + "val": { + "rmse": 0.5558022325623276, + "mae": 0.44188886880874634, + "r2": 0.4174697995185852, + "score": -0.5558022325623276 + }, + "test": { + "rmse": 0.5512480144508224, + "mae": 0.4457772672176361, + "r2": 0.4337567090988159, + "score": -0.5512480144508224 + } + }, + "time": "0:10:54.207858", + "diversity": { + "train": { + "mean_centered_corr": -0.019908307120203972, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07479876279830933 + }, + "val": { + "mean_centered_corr": -0.02006814442574978, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07624257355928421 + }, + "test": { + "mean_centered_corr": -0.019939016550779343, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07578995823860168 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0p5-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0p5-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0p5-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0p5-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..5be345413 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0p5-greedy-heads-evaluation/0/report.json @@ -0,0 +1,191 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5086880691661995, + "mae": 0.4017886221408844, + "r2": 0.5149915218353271, + "score": -0.5086880691661995 + }, + "val": { + "rmse": 0.5554104781133956, + "mae": 0.44194847345352173, + "r2": 0.4182906746864319, + "score": -0.5554104781133956 + }, + "test": { + "rmse": 0.5525622419967886, + "mae": 0.4474184215068817, + "r2": 0.4310535788536072, + "score": -0.5525622419967886 + } + }, + "time": "0:10:54.207858", + "diversity": { + "train": { + "mean_centered_corr": -0.019908307120203972, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07479876279830933 + }, + "val": { + "mean_centered_corr": -0.02006814442574978, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07624257355928421 + }, + "test": { + "mean_centered_corr": -0.019939016550779343, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07578995823860168 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 4, + 13, + 18, + 11, + 15, + 6, + 21, + 19, + 29, + 31, + 16, + 14, + 22 + ], + "head_selection_time": "0:00:00.393665" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0-best-head-evaluation/0/report.json new file mode 100644 index 000000000..368c5c71a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0-best-head-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5119769486010273, + "mae": 0.40593889355659485, + "r2": 0.5086996555328369, + "score": -0.5119769486010273 + }, + "val": { + "rmse": 0.5583888658501774, + "mae": 0.4450767934322357, + "r2": 0.41203510761260986, + "score": -0.5583888658501774 + }, + "test": { + "rmse": 0.5543243670215346, + "mae": 0.44926270842552185, + "r2": 0.4274190068244934, + "score": -0.5543243670215346 + } + }, + "time": "0:10:54.579717", + "diversity": { + "train": { + "mean_centered_corr": -0.02000253275036812, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07106093317270279 + }, + "val": { + "mean_centered_corr": -0.019867246970534325, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07228023558855057 + }, + "test": { + "mean_centered_corr": -0.020095540210604668, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07158966362476349 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 10 + ], + "head_selection_time": "0:00:00.023234" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0-evaluation/0.toml new file mode 100644 index 000000000..cc9c6b92e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.5 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0-evaluation/0/report.json new file mode 100644 index 000000000..83fb0c1c4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5068120050384366, + "mae": 0.3983139991760254, + "r2": 0.5185623168945312, + "score": -0.5068120050384366 + }, + "val": { + "rmse": 0.5557362219646558, + "mae": 0.4406058192253113, + "r2": 0.4176081418991089, + "score": -0.5557362219646558 + }, + "test": { + "rmse": 0.5505742164168008, + "mae": 0.4435580372810364, + "r2": 0.43514013290405273, + "score": -0.5505742164168008 + } + }, + "time": "0:10:54.579717", + "diversity": { + "train": { + "mean_centered_corr": -0.02000253275036812, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07106093317270279 + }, + "val": { + "mean_centered_corr": -0.019867246970534325, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07228023558855057 + }, + "test": { + "mean_centered_corr": -0.020095540210604668, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07158966362476349 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..dc718bf4b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0-greedy-heads-evaluation/0/report.json @@ -0,0 +1,194 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5072879723238778, + "mae": 0.39973556995391846, + "r2": 0.5176576375961304, + "score": -0.5072879723238778 + }, + "val": { + "rmse": 0.5551940861270195, + "mae": 0.44103240966796875, + "r2": 0.4187438488006592, + "score": -0.5551940861270195 + }, + "test": { + "rmse": 0.5519268755870242, + "mae": 0.44590064883232117, + "r2": 0.4323612451553345, + "score": -0.5519268755870242 + } + }, + "time": "0:10:54.579717", + "diversity": { + "train": { + "mean_centered_corr": -0.02000253275036812, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07106093317270279 + }, + "val": { + "mean_centered_corr": -0.019867246970534325, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07228023558855057 + }, + "test": { + "mean_centered_corr": -0.020095540210604668, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07158966362476349 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 10, + 13, + 21, + 26, + 25, + 18, + 14, + 6, + 11, + 29, + 13, + 4, + 19, + 21, + 16, + 26 + ], + "head_selection_time": "0:00:00.520164" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0p5-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0p5-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0p5-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0p5-best-head-evaluation/0/report.json new file mode 100644 index 000000000..3de887292 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0p5-best-head-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 3276, + "metrics": { + "train": { + "rmse": 0.502955286059696, + "mae": 0.3946622908115387, + "r2": 0.5258617401123047, + "score": -0.502955286059696 + }, + "val": { + "rmse": 0.5596610856119687, + "mae": 0.44277745485305786, + "r2": 0.4093528389930725, + "score": -0.5596610856119687 + }, + "test": { + "rmse": 0.5563749515907954, + "mae": 0.44843393564224243, + "r2": 0.4231749176979065, + "score": -0.5563749515907954 + } + }, + "time": "0:12:40.834182", + "diversity": { + "train": { + "mean_centered_corr": -0.020183589309453964, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0822177454829216 + }, + "val": { + "mean_centered_corr": -0.020031888037919998, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08821859210729599 + }, + "test": { + "mean_centered_corr": -0.01992768421769142, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08592649549245834 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023278" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0p5-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0p5-evaluation/0.toml new file mode 100644 index 000000000..ad04ceb1e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0p5-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.5 +rla_extra_paths_freeze_fraction = 0.5 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0p5-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0p5-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0p5-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0p5-evaluation/0/report.json new file mode 100644 index 000000000..0fb4e76c5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0p5-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 3276, + "metrics": { + "train": { + "rmse": 0.4956110911684622, + "mae": 0.3902001976966858, + "r2": 0.5396074056625366, + "score": -0.4956110911684622 + }, + "val": { + "rmse": 0.5558360123700841, + "mae": 0.43872880935668945, + "r2": 0.41739898920059204, + "score": -0.5558360123700841 + }, + "test": { + "rmse": 0.5491424149361828, + "mae": 0.4410218894481659, + "r2": 0.43807417154312134, + "score": -0.5491424149361828 + } + }, + "time": "0:12:40.834182", + "diversity": { + "train": { + "mean_centered_corr": -0.020183589309453964, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0822177454829216 + }, + "val": { + "mean_centered_corr": -0.020031888037919998, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08821859210729599 + }, + "test": { + "mean_centered_corr": -0.01992768421769142, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08592649549245834 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0p5-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0p5-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0p5-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0p5-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..343dbebb6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0p5-greedy-heads-evaluation/0/report.json @@ -0,0 +1,194 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 3276, + "metrics": { + "train": { + "rmse": 0.4960975196274642, + "mae": 0.3916766345500946, + "r2": 0.5387032628059387, + "score": -0.4960975196274642 + }, + "val": { + "rmse": 0.5552555453020565, + "mae": 0.4401964545249939, + "r2": 0.4186151623725891, + "score": -0.5552555453020565 + }, + "test": { + "rmse": 0.551575730248866, + "mae": 0.44500499963760376, + "r2": 0.43308329582214355, + "score": -0.551575730248866 + } + }, + "time": "0:12:40.834182", + "diversity": { + "train": { + "mean_centered_corr": -0.020183589309453964, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0822177454829216 + }, + "val": { + "mean_centered_corr": -0.020031888037919998, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08821859210729599 + }, + "test": { + "mean_centered_corr": -0.01992768421769142, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08592649549245834 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13, + 10, + 2, + 20, + 16, + 6, + 26, + 17, + 21, + 15, + 1, + 10, + 22, + 2, + 30, + 31 + ], + "head_selection_time": "0:00:00.523278" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0-best-head-evaluation/0/report.json new file mode 100644 index 000000000..d33af28cf --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0-best-head-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5159634144292107, + "mae": 0.410226047039032, + "r2": 0.5010189414024353, + "score": -0.5159634144292107 + }, + "val": { + "rmse": 0.5593939165294649, + "mae": 0.4475388526916504, + "r2": 0.40991663932800293, + "score": -0.5593939165294649 + }, + "test": { + "rmse": 0.5608508784321002, + "mae": 0.45661959052085876, + "r2": 0.41385674476623535, + "score": -0.5608508784321002 + } + }, + "time": "0:16:51.845095", + "diversity": { + "train": { + "mean_centered_corr": -0.019908307120203972, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07479876279830933 + }, + "val": { + "mean_centered_corr": -0.02006814442574978, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07624257355928421 + }, + "test": { + "mean_centered_corr": -0.019939016550779343, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07578995823860168 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.023303" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0-evaluation/0.toml new file mode 100644 index 000000000..815cb9308 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.25 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0-evaluation/0/report.json new file mode 100644 index 000000000..f3e58291a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5088939009594995, + "mae": 0.40171560645103455, + "r2": 0.5145988464355469, + "score": -0.5088939009594995 + }, + "val": { + "rmse": 0.5558022325623276, + "mae": 0.44188886880874634, + "r2": 0.4174697995185852, + "score": -0.5558022325623276 + }, + "test": { + "rmse": 0.5512480144508224, + "mae": 0.4457772672176361, + "r2": 0.4337567090988159, + "score": -0.5512480144508224 + } + }, + "time": "0:16:51.845095", + "diversity": { + "train": { + "mean_centered_corr": -0.019908307120203972, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07479876279830933 + }, + "val": { + "mean_centered_corr": -0.02006814442574978, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07624257355928421 + }, + "test": { + "mean_centered_corr": -0.019939016550779343, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07578995823860168 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..3b7b383d1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0-greedy-heads-evaluation/0/report.json @@ -0,0 +1,191 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5086880691661995, + "mae": 0.4017886221408844, + "r2": 0.5149915218353271, + "score": -0.5086880691661995 + }, + "val": { + "rmse": 0.5554104781133956, + "mae": 0.44194847345352173, + "r2": 0.4182906746864319, + "score": -0.5554104781133956 + }, + "test": { + "rmse": 0.5525622419967886, + "mae": 0.4474184215068817, + "r2": 0.4310535788536072, + "score": -0.5525622419967886 + } + }, + "time": "0:16:51.845095", + "diversity": { + "train": { + "mean_centered_corr": -0.019908307120203972, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07479876279830933 + }, + "val": { + "mean_centered_corr": -0.02006814442574978, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07624257355928421 + }, + "test": { + "mean_centered_corr": -0.019939016550779343, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07578995823860168 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 4, + 13, + 18, + 11, + 15, + 6, + 21, + 19, + 29, + 31, + 16, + 14, + 22 + ], + "head_selection_time": "0:00:00.393834" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0p5-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0p5-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0p5-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0p5-best-head-evaluation/0/report.json new file mode 100644 index 000000000..4d4007f7b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0p5-best-head-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5159634144292107, + "mae": 0.410226047039032, + "r2": 0.5010189414024353, + "score": -0.5159634144292107 + }, + "val": { + "rmse": 0.5593939165294649, + "mae": 0.4475388526916504, + "r2": 0.40991663932800293, + "score": -0.5593939165294649 + }, + "test": { + "rmse": 0.5608508784321002, + "mae": 0.45661959052085876, + "r2": 0.41385674476623535, + "score": -0.5608508784321002 + } + }, + "time": "0:16:52.052732", + "diversity": { + "train": { + "mean_centered_corr": -0.019908307120203972, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07479876279830933 + }, + "val": { + "mean_centered_corr": -0.02006814442574978, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07624257355928421 + }, + "test": { + "mean_centered_corr": -0.019939016550779343, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07578995823860168 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.023593" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0p5-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0p5-evaluation/0.toml new file mode 100644 index 000000000..88ef7e1b1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0p5-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.25 +rla_extra_paths_freeze_fraction = 0.5 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0p5-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0p5-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0p5-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0p5-evaluation/0/report.json new file mode 100644 index 000000000..cdb6dbee0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0p5-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5088939009594995, + "mae": 0.40171560645103455, + "r2": 0.5145988464355469, + "score": -0.5088939009594995 + }, + "val": { + "rmse": 0.5558022325623276, + "mae": 0.44188886880874634, + "r2": 0.4174697995185852, + "score": -0.5558022325623276 + }, + "test": { + "rmse": 0.5512480144508224, + "mae": 0.4457772672176361, + "r2": 0.4337567090988159, + "score": -0.5512480144508224 + } + }, + "time": "0:16:52.052732", + "diversity": { + "train": { + "mean_centered_corr": -0.019908307120203972, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07479876279830933 + }, + "val": { + "mean_centered_corr": -0.02006814442574978, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07624257355928421 + }, + "test": { + "mean_centered_corr": -0.019939016550779343, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07578995823860168 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0p5-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0p5-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0p5-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0p5-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..2913e4690 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0p5-greedy-heads-evaluation/0/report.json @@ -0,0 +1,191 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5086880691661995, + "mae": 0.4017886221408844, + "r2": 0.5149915218353271, + "score": -0.5086880691661995 + }, + "val": { + "rmse": 0.5554104781133956, + "mae": 0.44194847345352173, + "r2": 0.4182906746864319, + "score": -0.5554104781133956 + }, + "test": { + "rmse": 0.5525622419967886, + "mae": 0.4474184215068817, + "r2": 0.4310535788536072, + "score": -0.5525622419967886 + } + }, + "time": "0:16:52.052732", + "diversity": { + "train": { + "mean_centered_corr": -0.019908307120203972, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07479876279830933 + }, + "val": { + "mean_centered_corr": -0.02006814442574978, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07624257355928421 + }, + "test": { + "mean_centered_corr": -0.019939016550779343, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07578995823860168 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 4, + 13, + 18, + 11, + 15, + 6, + 21, + 19, + 29, + 31, + 16, + 14, + 22 + ], + "head_selection_time": "0:00:00.408183" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0-best-head-evaluation/0/report.json new file mode 100644 index 000000000..ff9ee8395 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0-best-head-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 3276, + "metrics": { + "train": { + "rmse": 0.502955286059696, + "mae": 0.3946622908115387, + "r2": 0.5258617401123047, + "score": -0.502955286059696 + }, + "val": { + "rmse": 0.5596610856119687, + "mae": 0.44277745485305786, + "r2": 0.4093528389930725, + "score": -0.5596610856119687 + }, + "test": { + "rmse": 0.5563749515907954, + "mae": 0.44843393564224243, + "r2": 0.4231749176979065, + "score": -0.5563749515907954 + } + }, + "time": "0:19:39.258488", + "diversity": { + "train": { + "mean_centered_corr": -0.020183589309453964, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0822177454829216 + }, + "val": { + "mean_centered_corr": -0.020031888037919998, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08821859210729599 + }, + "test": { + "mean_centered_corr": -0.01992768421769142, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08592649549245834 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023852" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0-evaluation/0.toml new file mode 100644 index 000000000..6025ce67d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.5 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0-evaluation/0/report.json new file mode 100644 index 000000000..b30eebcd1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 3276, + "metrics": { + "train": { + "rmse": 0.4956110911684622, + "mae": 0.3902001976966858, + "r2": 0.5396074056625366, + "score": -0.4956110911684622 + }, + "val": { + "rmse": 0.5558360123700841, + "mae": 0.43872880935668945, + "r2": 0.41739898920059204, + "score": -0.5558360123700841 + }, + "test": { + "rmse": 0.5491424149361828, + "mae": 0.4410218894481659, + "r2": 0.43807417154312134, + "score": -0.5491424149361828 + } + }, + "time": "0:19:39.258488", + "diversity": { + "train": { + "mean_centered_corr": -0.020183589309453964, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0822177454829216 + }, + "val": { + "mean_centered_corr": -0.020031888037919998, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08821859210729599 + }, + "test": { + "mean_centered_corr": -0.01992768421769142, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08592649549245834 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..f1cf12fa3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0-greedy-heads-evaluation/0/report.json @@ -0,0 +1,194 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 3276, + "metrics": { + "train": { + "rmse": 0.4960975196274642, + "mae": 0.3916766345500946, + "r2": 0.5387032628059387, + "score": -0.4960975196274642 + }, + "val": { + "rmse": 0.5552555453020565, + "mae": 0.4401964545249939, + "r2": 0.4186151623725891, + "score": -0.5552555453020565 + }, + "test": { + "rmse": 0.551575730248866, + "mae": 0.44500499963760376, + "r2": 0.43308329582214355, + "score": -0.551575730248866 + } + }, + "time": "0:19:39.258488", + "diversity": { + "train": { + "mean_centered_corr": -0.020183589309453964, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0822177454829216 + }, + "val": { + "mean_centered_corr": -0.020031888037919998, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08821859210729599 + }, + "test": { + "mean_centered_corr": -0.01992768421769142, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08592649549245834 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13, + 10, + 2, + 20, + 16, + 6, + 26, + 17, + 21, + 15, + 1, + 10, + 22, + 2, + 30, + 31 + ], + "head_selection_time": "0:00:00.492201" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0p5-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0p5-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0p5-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0p5-best-head-evaluation/0/report.json new file mode 100644 index 000000000..cbe0b2a93 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0p5-best-head-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 3276, + "metrics": { + "train": { + "rmse": 0.502955286059696, + "mae": 0.3946622908115387, + "r2": 0.5258617401123047, + "score": -0.502955286059696 + }, + "val": { + "rmse": 0.5596610856119687, + "mae": 0.44277745485305786, + "r2": 0.4093528389930725, + "score": -0.5596610856119687 + }, + "test": { + "rmse": 0.5563749515907954, + "mae": 0.44843393564224243, + "r2": 0.4231749176979065, + "score": -0.5563749515907954 + } + }, + "time": "0:19:40.313994", + "diversity": { + "train": { + "mean_centered_corr": -0.020183589309453964, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0822177454829216 + }, + "val": { + "mean_centered_corr": -0.020031888037919998, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08821859210729599 + }, + "test": { + "mean_centered_corr": -0.01992768421769142, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08592649549245834 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023504" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0p5-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0p5-evaluation/0.toml new file mode 100644 index 000000000..57515fe2a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0p5-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.5 +rla_extra_paths_freeze_fraction = 0.5 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0p5-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0p5-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0p5-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0p5-evaluation/0/report.json new file mode 100644 index 000000000..c295956d9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0p5-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 3276, + "metrics": { + "train": { + "rmse": 0.4956110911684622, + "mae": 0.3902001976966858, + "r2": 0.5396074056625366, + "score": -0.4956110911684622 + }, + "val": { + "rmse": 0.5558360123700841, + "mae": 0.43872880935668945, + "r2": 0.41739898920059204, + "score": -0.5558360123700841 + }, + "test": { + "rmse": 0.5491424149361828, + "mae": 0.4410218894481659, + "r2": 0.43807417154312134, + "score": -0.5491424149361828 + } + }, + "time": "0:19:40.313994", + "diversity": { + "train": { + "mean_centered_corr": -0.020183589309453964, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0822177454829216 + }, + "val": { + "mean_centered_corr": -0.020031888037919998, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08821859210729599 + }, + "test": { + "mean_centered_corr": -0.01992768421769142, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08592649549245834 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0p5-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0p5-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0p5-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0p5-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..63c75d98f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0p5-greedy-heads-evaluation/0/report.json @@ -0,0 +1,194 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 3276, + "metrics": { + "train": { + "rmse": 0.4960975196274642, + "mae": 0.3916766345500946, + "r2": 0.5387032628059387, + "score": -0.4960975196274642 + }, + "val": { + "rmse": 0.5552555453020565, + "mae": 0.4401964545249939, + "r2": 0.4186151623725891, + "score": -0.5552555453020565 + }, + "test": { + "rmse": 0.551575730248866, + "mae": 0.44500499963760376, + "r2": 0.43308329582214355, + "score": -0.551575730248866 + } + }, + "time": "0:19:40.313994", + "diversity": { + "train": { + "mean_centered_corr": -0.020183589309453964, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0822177454829216 + }, + "val": { + "mean_centered_corr": -0.020031888037919998, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08821859210729599 + }, + "test": { + "mean_centered_corr": -0.01992768421769142, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08592649549245834 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13, + 10, + 2, + 20, + 16, + 6, + 26, + 17, + 21, + 15, + 1, + 10, + 22, + 2, + 30, + 31 + ], + "head_selection_time": "0:00:00.510200" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0-best-head-evaluation/0/report.json new file mode 100644 index 000000000..166c52623 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0-best-head-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 3276, + "metrics": { + "train": { + "rmse": 0.5089933603917912, + "mae": 0.4041629731655121, + "r2": 0.5144091844558716, + "score": -0.5089933603917912 + }, + "val": { + "rmse": 0.56004920160056, + "mae": 0.44279050827026367, + "r2": 0.40853333473205566, + "score": -0.56004920160056 + }, + "test": { + "rmse": 0.5537449525211285, + "mae": 0.4460917115211487, + "r2": 0.4286153316497803, + "score": -0.5537449525211285 + } + }, + "time": "0:19:35.794115", + "diversity": { + "train": { + "mean_centered_corr": -0.019954748451709747, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08140051364898682 + }, + "val": { + "mean_centered_corr": -0.020047752186655998, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0873202458024025 + }, + "test": { + "mean_centered_corr": -0.019868871197104454, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08496350795030594 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 18 + ], + "head_selection_time": "0:00:00.024205" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0-evaluation/0.toml new file mode 100644 index 000000000..5c7b4aa29 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.25 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0-evaluation/0/report.json new file mode 100644 index 000000000..fbc6695c7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 3276, + "metrics": { + "train": { + "rmse": 0.4982967871668599, + "mae": 0.3913329243659973, + "r2": 0.5346041917800903, + "score": -0.4982967871668599 + }, + "val": { + "rmse": 0.5564917917545931, + "mae": 0.4382628798484802, + "r2": 0.4160234332084656, + "score": -0.5564917917545931 + }, + "test": { + "rmse": 0.5481810097693468, + "mae": 0.4391641318798065, + "r2": 0.44004005193710327, + "score": -0.5481810097693468 + } + }, + "time": "0:19:35.794115", + "diversity": { + "train": { + "mean_centered_corr": -0.019954748451709747, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08140051364898682 + }, + "val": { + "mean_centered_corr": -0.020047752186655998, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0873202458024025 + }, + "test": { + "mean_centered_corr": -0.019868871197104454, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08496350795030594 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..15592bdf7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0-greedy-heads-evaluation/0/report.json @@ -0,0 +1,189 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 3276, + "metrics": { + "train": { + "rmse": 0.4995550321442935, + "mae": 0.3944211006164551, + "r2": 0.5322508811950684, + "score": -0.4995550321442935 + }, + "val": { + "rmse": 0.5553600907849848, + "mae": 0.4404390752315521, + "r2": 0.4183961749076843, + "score": -0.5553600907849848 + }, + "test": { + "rmse": 0.5504577714884789, + "mae": 0.44397687911987305, + "r2": 0.4353790283203125, + "score": -0.5504577714884789 + } + }, + "time": "0:19:35.794115", + "diversity": { + "train": { + "mean_centered_corr": -0.019954748451709747, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08140051364898682 + }, + "val": { + "mean_centered_corr": -0.020047752186655998, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0873202458024025 + }, + "test": { + "mean_centered_corr": -0.019868871197104454, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08496350795030594 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 18, + 1, + 12, + 5, + 25, + 13, + 14, + 15, + 6, + 0, + 11 + ], + "head_selection_time": "0:00:00.337718" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0p5-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0p5-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0p5-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0p5-best-head-evaluation/0/report.json new file mode 100644 index 000000000..ea89cd2b5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0p5-best-head-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5159634144292107, + "mae": 0.410226047039032, + "r2": 0.5010189414024353, + "score": -0.5159634144292107 + }, + "val": { + "rmse": 0.5593939165294649, + "mae": 0.4475388526916504, + "r2": 0.40991663932800293, + "score": -0.5593939165294649 + }, + "test": { + "rmse": 0.5608508784321002, + "mae": 0.45661959052085876, + "r2": 0.41385674476623535, + "score": -0.5608508784321002 + } + }, + "time": "0:16:49.735827", + "diversity": { + "train": { + "mean_centered_corr": -0.019908307120203972, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07479876279830933 + }, + "val": { + "mean_centered_corr": -0.02006814442574978, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07624257355928421 + }, + "test": { + "mean_centered_corr": -0.019939016550779343, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07578995823860168 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.023311" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0p5-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0p5-evaluation/0.toml new file mode 100644 index 000000000..3cf5b599b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0p5-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.25 +rla_extra_paths_freeze_fraction = 0.5 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0p5-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0p5-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0p5-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0p5-evaluation/0/report.json new file mode 100644 index 000000000..b587ef7c9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0p5-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5088939009594995, + "mae": 0.40171560645103455, + "r2": 0.5145988464355469, + "score": -0.5088939009594995 + }, + "val": { + "rmse": 0.5558022325623276, + "mae": 0.44188886880874634, + "r2": 0.4174697995185852, + "score": -0.5558022325623276 + }, + "test": { + "rmse": 0.5512480144508224, + "mae": 0.4457772672176361, + "r2": 0.4337567090988159, + "score": -0.5512480144508224 + } + }, + "time": "0:16:49.735827", + "diversity": { + "train": { + "mean_centered_corr": -0.019908307120203972, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07479876279830933 + }, + "val": { + "mean_centered_corr": -0.02006814442574978, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07624257355928421 + }, + "test": { + "mean_centered_corr": -0.019939016550779343, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07578995823860168 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0p5-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0p5-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0p5-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0p5-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..83873e8d4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0p5-greedy-heads-evaluation/0/report.json @@ -0,0 +1,191 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5086880691661995, + "mae": 0.4017886221408844, + "r2": 0.5149915218353271, + "score": -0.5086880691661995 + }, + "val": { + "rmse": 0.5554104781133956, + "mae": 0.44194847345352173, + "r2": 0.4182906746864319, + "score": -0.5554104781133956 + }, + "test": { + "rmse": 0.5525622419967886, + "mae": 0.4474184215068817, + "r2": 0.4310535788536072, + "score": -0.5525622419967886 + } + }, + "time": "0:16:49.735827", + "diversity": { + "train": { + "mean_centered_corr": -0.019908307120203972, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07479876279830933 + }, + "val": { + "mean_centered_corr": -0.02006814442574978, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07624257355928421 + }, + "test": { + "mean_centered_corr": -0.019939016550779343, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07578995823860168 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 4, + 13, + 18, + 11, + 15, + 6, + 21, + 19, + 29, + 31, + 16, + 14, + 22 + ], + "head_selection_time": "0:00:00.404210" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0-best-head-evaluation/0/report.json new file mode 100644 index 000000000..e5b025742 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0-best-head-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5175918577279733, + "mae": 0.40775081515312195, + "r2": 0.4978642463684082, + "score": -0.5175918577279733 + }, + "val": { + "rmse": 0.5572652691682626, + "mae": 0.44076380133628845, + "r2": 0.4143989086151123, + "score": -0.5572652691682626 + }, + "test": { + "rmse": 0.5524207799310179, + "mae": 0.4452172815799713, + "r2": 0.4313448667526245, + "score": -0.5524207799310179 + } + }, + "time": "0:14:43.657532", + "diversity": { + "train": { + "mean_centered_corr": -0.02016017585992813, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06105273962020874 + }, + "val": { + "mean_centered_corr": -0.01993357017636299, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06221357360482216 + }, + "test": { + "mean_centered_corr": -0.020221160724759102, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06099524348974228 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 11 + ], + "head_selection_time": "0:00:00.023842" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0-evaluation/0.toml new file mode 100644 index 000000000..f15a53b07 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.5 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0-evaluation/0/report.json new file mode 100644 index 000000000..ddf9415fb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5150130136938161, + "mae": 0.40628108382225037, + "r2": 0.5028554797172546, + "score": -0.5150130136938161 + }, + "val": { + "rmse": 0.5559356776481553, + "mae": 0.441616415977478, + "r2": 0.4171900153160095, + "score": -0.5559356776481553 + }, + "test": { + "rmse": 0.55135486033622, + "mae": 0.445721298456192, + "r2": 0.43353718519210815, + "score": -0.55135486033622 + } + }, + "time": "0:14:43.657532", + "diversity": { + "train": { + "mean_centered_corr": -0.02016017585992813, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06105273962020874 + }, + "val": { + "mean_centered_corr": -0.01993357017636299, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06221357360482216 + }, + "test": { + "mean_centered_corr": -0.020221160724759102, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06099524348974228 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..f95c28851 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0-greedy-heads-evaluation/0/report.json @@ -0,0 +1,187 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5148489632002486, + "mae": 0.40648993849754333, + "r2": 0.5031721591949463, + "score": -0.5148489632002486 + }, + "val": { + "rmse": 0.5553930389270275, + "mae": 0.4419495463371277, + "r2": 0.4183272123336792, + "score": -0.5553930389270275 + }, + "test": { + "rmse": 0.5519483119271499, + "mae": 0.4472074508666992, + "r2": 0.4323171377182007, + "score": -0.5519483119271499 + } + }, + "time": "0:14:43.657532", + "diversity": { + "train": { + "mean_centered_corr": -0.02016017585992813, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06105273962020874 + }, + "val": { + "mean_centered_corr": -0.01993357017636299, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06221357360482216 + }, + "test": { + "mean_centered_corr": -0.020221160724759102, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06099524348974228 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 11, + 10, + 26, + 21, + 15, + 16, + 31, + 19, + 13 + ], + "head_selection_time": "0:00:00.256647" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0p5-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0p5-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0p5-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0p5-best-head-evaluation/0/report.json new file mode 100644 index 000000000..48e0ae5f6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0p5-best-head-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 3276, + "metrics": { + "train": { + "rmse": 0.502955286059696, + "mae": 0.3946622908115387, + "r2": 0.5258617401123047, + "score": -0.502955286059696 + }, + "val": { + "rmse": 0.5596610856119687, + "mae": 0.44277745485305786, + "r2": 0.4093528389930725, + "score": -0.5596610856119687 + }, + "test": { + "rmse": 0.5563749515907954, + "mae": 0.44843393564224243, + "r2": 0.4231749176979065, + "score": -0.5563749515907954 + } + }, + "time": "0:19:37.834703", + "diversity": { + "train": { + "mean_centered_corr": -0.020183589309453964, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0822177454829216 + }, + "val": { + "mean_centered_corr": -0.020031888037919998, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08821859210729599 + }, + "test": { + "mean_centered_corr": -0.01992768421769142, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08592649549245834 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.024218" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0p5-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0p5-evaluation/0.toml new file mode 100644 index 000000000..46d1d4c33 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0p5-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.5 +rla_extra_paths_freeze_fraction = 0.5 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0p5-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0p5-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0p5-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0p5-evaluation/0/report.json new file mode 100644 index 000000000..1dbd8a906 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0p5-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 3276, + "metrics": { + "train": { + "rmse": 0.4956110911684622, + "mae": 0.3902001976966858, + "r2": 0.5396074056625366, + "score": -0.4956110911684622 + }, + "val": { + "rmse": 0.5558360123700841, + "mae": 0.43872880935668945, + "r2": 0.41739898920059204, + "score": -0.5558360123700841 + }, + "test": { + "rmse": 0.5491424149361828, + "mae": 0.4410218894481659, + "r2": 0.43807417154312134, + "score": -0.5491424149361828 + } + }, + "time": "0:19:37.834703", + "diversity": { + "train": { + "mean_centered_corr": -0.020183589309453964, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0822177454829216 + }, + "val": { + "mean_centered_corr": -0.020031888037919998, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08821859210729599 + }, + "test": { + "mean_centered_corr": -0.01992768421769142, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08592649549245834 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0p5-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0p5-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0p5-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0p5-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..0ccda495a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0p5-greedy-heads-evaluation/0/report.json @@ -0,0 +1,194 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 3276, + "metrics": { + "train": { + "rmse": 0.4960975196274642, + "mae": 0.3916766345500946, + "r2": 0.5387032628059387, + "score": -0.4960975196274642 + }, + "val": { + "rmse": 0.5552555453020565, + "mae": 0.4401964545249939, + "r2": 0.4186151623725891, + "score": -0.5552555453020565 + }, + "test": { + "rmse": 0.551575730248866, + "mae": 0.44500499963760376, + "r2": 0.43308329582214355, + "score": -0.551575730248866 + } + }, + "time": "0:19:37.834703", + "diversity": { + "train": { + "mean_centered_corr": -0.020183589309453964, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0822177454829216 + }, + "val": { + "mean_centered_corr": -0.020031888037919998, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08821859210729599 + }, + "test": { + "mean_centered_corr": -0.01992768421769142, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08592649549245834 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13, + 10, + 2, + 20, + 16, + 6, + 26, + 17, + 21, + 15, + 1, + 10, + 22, + 2, + 30, + 31 + ], + "head_selection_time": "0:00:00.498576" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0-best-head-evaluation/0/report.json new file mode 100644 index 000000000..34f9c8081 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0-best-head-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 3276, + "metrics": { + "train": { + "rmse": 0.5111900976758429, + "mae": 0.40660035610198975, + "r2": 0.5102086067199707, + "score": -0.5111900976758429 + }, + "val": { + "rmse": 0.5601935253217213, + "mae": 0.4456070363521576, + "r2": 0.40822845697402954, + "score": -0.5601935253217213 + }, + "test": { + "rmse": 0.5598375837203123, + "mae": 0.45322397351264954, + "r2": 0.4159727692604065, + "score": -0.5598375837203123 + } + }, + "time": "0:19:35.624704", + "diversity": { + "train": { + "mean_centered_corr": -0.02013956569135189, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0812845528125763 + }, + "val": { + "mean_centered_corr": -0.02026885561645031, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08767195791006088 + }, + "test": { + "mean_centered_corr": -0.019879044964909554, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08455239981412888 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 1 + ], + "head_selection_time": "0:00:00.023504" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0-evaluation/0.toml new file mode 100644 index 000000000..85fb0b1da --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.25 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0-evaluation/0/report.json new file mode 100644 index 000000000..20fc4d6d9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 3276, + "metrics": { + "train": { + "rmse": 0.4985970842566162, + "mae": 0.3925943076610565, + "r2": 0.5340430736541748, + "score": -0.4985970842566162 + }, + "val": { + "rmse": 0.5560380585478252, + "mae": 0.43930554389953613, + "r2": 0.41697531938552856, + "score": -0.5560380585478252 + }, + "test": { + "rmse": 0.5494440227195571, + "mae": 0.44151392579078674, + "r2": 0.43745678663253784, + "score": -0.5494440227195571 + } + }, + "time": "0:19:35.624704", + "diversity": { + "train": { + "mean_centered_corr": -0.02013956569135189, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0812845528125763 + }, + "val": { + "mean_centered_corr": -0.02026885561645031, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08767195791006088 + }, + "test": { + "mean_centered_corr": -0.019879044964909554, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08455239981412888 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..0ec7829f3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0-greedy-heads-evaluation/0/report.json @@ -0,0 +1,190 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 3276, + "metrics": { + "train": { + "rmse": 0.4998972608393945, + "mae": 0.39439302682876587, + "r2": 0.5316097736358643, + "score": -0.4998972608393945 + }, + "val": { + "rmse": 0.5553281335127394, + "mae": 0.4403950870037079, + "r2": 0.4184631109237671, + "score": -0.5553281335127394 + }, + "test": { + "rmse": 0.550823697016921, + "mae": 0.44420838356018066, + "r2": 0.43462812900543213, + "score": -0.550823697016921 + } + }, + "time": "0:19:35.624704", + "diversity": { + "train": { + "mean_centered_corr": -0.02013956569135189, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0812845528125763 + }, + "val": { + "mean_centered_corr": -0.02026885561645031, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08767195791006088 + }, + "test": { + "mean_centered_corr": -0.019879044964909554, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08455239981412888 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 1, + 18, + 29, + 14, + 11, + 20, + 10, + 13, + 25, + 12, + 6, + 26 + ], + "head_selection_time": "0:00:00.359737" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0p5-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0p5-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0p5-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0p5-best-head-evaluation/0/report.json new file mode 100644 index 000000000..ed541478f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0p5-best-head-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5159634144292107, + "mae": 0.410226047039032, + "r2": 0.5010189414024353, + "score": -0.5159634144292107 + }, + "val": { + "rmse": 0.5593939165294649, + "mae": 0.4475388526916504, + "r2": 0.40991663932800293, + "score": -0.5593939165294649 + }, + "test": { + "rmse": 0.5608508784321002, + "mae": 0.45661959052085876, + "r2": 0.41385674476623535, + "score": -0.5608508784321002 + } + }, + "time": "0:16:46.550435", + "diversity": { + "train": { + "mean_centered_corr": -0.019908307120203972, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07479876279830933 + }, + "val": { + "mean_centered_corr": -0.02006814442574978, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07624257355928421 + }, + "test": { + "mean_centered_corr": -0.019939016550779343, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07578995823860168 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.023743" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0p5-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0p5-evaluation/0.toml new file mode 100644 index 000000000..a3c652359 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0p5-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.25 +rla_extra_paths_freeze_fraction = 0.5 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0p5-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0p5-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0p5-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0p5-evaluation/0/report.json new file mode 100644 index 000000000..f72692cdd --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0p5-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5088939009594995, + "mae": 0.40171560645103455, + "r2": 0.5145988464355469, + "score": -0.5088939009594995 + }, + "val": { + "rmse": 0.5558022325623276, + "mae": 0.44188886880874634, + "r2": 0.4174697995185852, + "score": -0.5558022325623276 + }, + "test": { + "rmse": 0.5512480144508224, + "mae": 0.4457772672176361, + "r2": 0.4337567090988159, + "score": -0.5512480144508224 + } + }, + "time": "0:16:46.550435", + "diversity": { + "train": { + "mean_centered_corr": -0.019908307120203972, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07479876279830933 + }, + "val": { + "mean_centered_corr": -0.02006814442574978, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07624257355928421 + }, + "test": { + "mean_centered_corr": -0.019939016550779343, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07578995823860168 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0p5-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0p5-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0p5-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0p5-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..2224a0377 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0p5-greedy-heads-evaluation/0/report.json @@ -0,0 +1,191 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5086880691661995, + "mae": 0.4017886221408844, + "r2": 0.5149915218353271, + "score": -0.5086880691661995 + }, + "val": { + "rmse": 0.5554104781133956, + "mae": 0.44194847345352173, + "r2": 0.4182906746864319, + "score": -0.5554104781133956 + }, + "test": { + "rmse": 0.5525622419967886, + "mae": 0.4474184215068817, + "r2": 0.4310535788536072, + "score": -0.5525622419967886 + } + }, + "time": "0:16:46.550435", + "diversity": { + "train": { + "mean_centered_corr": -0.019908307120203972, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07479876279830933 + }, + "val": { + "mean_centered_corr": -0.02006814442574978, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07624257355928421 + }, + "test": { + "mean_centered_corr": -0.019939016550779343, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07578995823860168 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 4, + 13, + 18, + 11, + 15, + 6, + 21, + 19, + 29, + 31, + 16, + 14, + 22 + ], + "head_selection_time": "0:00:00.397701" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0-best-head-evaluation/0/report.json new file mode 100644 index 000000000..7e807e060 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0-best-head-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.516478064040768, + "mae": 0.40249156951904297, + "r2": 0.5000230073928833, + "score": -0.516478064040768 + }, + "val": { + "rmse": 0.5579574268466545, + "mae": 0.4429668188095093, + "r2": 0.41294336318969727, + "score": -0.5579574268466545 + }, + "test": { + "rmse": 0.5527197091141781, + "mae": 0.44649186730384827, + "r2": 0.4307292699813843, + "score": -0.5527197091141781 + } + }, + "time": "0:14:45.943513", + "diversity": { + "train": { + "mean_centered_corr": -0.020442524924874306, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06176675111055374 + }, + "val": { + "mean_centered_corr": -0.020368289202451706, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06270569562911987 + }, + "test": { + "mean_centered_corr": -0.020271461457014084, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06161132827401161 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023288" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0-evaluation/0.toml new file mode 100644 index 000000000..e2f18e728 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.5 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0-evaluation/0/report.json new file mode 100644 index 000000000..138ab14aa --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5141129141438994, + "mae": 0.4034823179244995, + "r2": 0.504591703414917, + "score": -0.5141129141438994 + }, + "val": { + "rmse": 0.5560025757843153, + "mae": 0.44079530239105225, + "r2": 0.41704970598220825, + "score": -0.5560025757843153 + }, + "test": { + "rmse": 0.550425719096006, + "mae": 0.44393521547317505, + "r2": 0.43544477224349976, + "score": -0.550425719096006 + } + }, + "time": "0:14:45.943513", + "diversity": { + "train": { + "mean_centered_corr": -0.020442524924874306, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06176675111055374 + }, + "val": { + "mean_centered_corr": -0.020368289202451706, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06270569562911987 + }, + "test": { + "mean_centered_corr": -0.020271461457014084, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06161132827401161 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..75cd99ca0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0-greedy-heads-evaluation/0/report.json @@ -0,0 +1,187 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5144092503450841, + "mae": 0.40473756194114685, + "r2": 0.5040204524993896, + "score": -0.5144092503450841 + }, + "val": { + "rmse": 0.5553178831836687, + "mae": 0.44113633036613464, + "r2": 0.418484628200531, + "score": -0.5553178831836687 + }, + "test": { + "rmse": 0.5514062081885815, + "mae": 0.4456017017364502, + "r2": 0.4334316849708557, + "score": -0.5514062081885815 + } + }, + "time": "0:14:45.943513", + "diversity": { + "train": { + "mean_centered_corr": -0.020442524924874306, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06176675111055374 + }, + "val": { + "mean_centered_corr": -0.020368289202451706, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06270569562911987 + }, + "test": { + "mean_centered_corr": -0.020271461457014084, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06161132827401161 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13, + 15, + 11, + 14, + 26, + 10, + 31, + 16, + 6 + ], + "head_selection_time": "0:00:00.258774" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0p5-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0p5-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0p5-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0p5-best-head-evaluation/0/report.json new file mode 100644 index 000000000..e4887b5bf --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0p5-best-head-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 3276, + "metrics": { + "train": { + "rmse": 0.502955286059696, + "mae": 0.3946622908115387, + "r2": 0.5258617401123047, + "score": -0.502955286059696 + }, + "val": { + "rmse": 0.5596610856119687, + "mae": 0.44277745485305786, + "r2": 0.4093528389930725, + "score": -0.5596610856119687 + }, + "test": { + "rmse": 0.5563749515907954, + "mae": 0.44843393564224243, + "r2": 0.4231749176979065, + "score": -0.5563749515907954 + } + }, + "time": "0:19:38.897747", + "diversity": { + "train": { + "mean_centered_corr": -0.020183589309453964, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0822177454829216 + }, + "val": { + "mean_centered_corr": -0.020031888037919998, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08821859210729599 + }, + "test": { + "mean_centered_corr": -0.01992768421769142, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08592649549245834 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023489" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0p5-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0p5-evaluation/0.toml new file mode 100644 index 000000000..58e4ac18e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0p5-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.5 +rla_extra_paths_freeze_fraction = 0.5 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0p5-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0p5-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0p5-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0p5-evaluation/0/report.json new file mode 100644 index 000000000..6a3c21869 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0p5-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 3276, + "metrics": { + "train": { + "rmse": 0.4956110911684622, + "mae": 0.3902001976966858, + "r2": 0.5396074056625366, + "score": -0.4956110911684622 + }, + "val": { + "rmse": 0.5558360123700841, + "mae": 0.43872880935668945, + "r2": 0.41739898920059204, + "score": -0.5558360123700841 + }, + "test": { + "rmse": 0.5491424149361828, + "mae": 0.4410218894481659, + "r2": 0.43807417154312134, + "score": -0.5491424149361828 + } + }, + "time": "0:19:38.897747", + "diversity": { + "train": { + "mean_centered_corr": -0.020183589309453964, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0822177454829216 + }, + "val": { + "mean_centered_corr": -0.020031888037919998, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08821859210729599 + }, + "test": { + "mean_centered_corr": -0.01992768421769142, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08592649549245834 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0p5-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0p5-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0p5-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0p5-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..d2d3dff39 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0p5-greedy-heads-evaluation/0/report.json @@ -0,0 +1,194 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 3276, + "metrics": { + "train": { + "rmse": 0.4960975196274642, + "mae": 0.3916766345500946, + "r2": 0.5387032628059387, + "score": -0.4960975196274642 + }, + "val": { + "rmse": 0.5552555453020565, + "mae": 0.4401964545249939, + "r2": 0.4186151623725891, + "score": -0.5552555453020565 + }, + "test": { + "rmse": 0.551575730248866, + "mae": 0.44500499963760376, + "r2": 0.43308329582214355, + "score": -0.551575730248866 + } + }, + "time": "0:19:38.897747", + "diversity": { + "train": { + "mean_centered_corr": -0.020183589309453964, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0822177454829216 + }, + "val": { + "mean_centered_corr": -0.020031888037919998, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08821859210729599 + }, + "test": { + "mean_centered_corr": -0.01992768421769142, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08592649549245834 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13, + 10, + 2, + 20, + 16, + 6, + 26, + 17, + 21, + 15, + 1, + 10, + 22, + 2, + 30, + 31 + ], + "head_selection_time": "0:00:00.511746" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..26a79c999 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,652 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2282046786382406, + "mae": 0.12737023830413818, + "r2": 0.6672922372817993, + "score": -0.2282046786382406 + }, + "val": { + "rmse": 0.2390544572113064, + "mae": 0.14095881581306458, + "r2": 0.6182799935340881, + "score": -0.2390544572113064 + }, + "test": { + "rmse": 0.25549556988031324, + "mae": 0.17997059226036072, + "r2": 0.5539265871047974, + "score": -0.25549556988031324 + } + }, + "time": "0:01:08.154288", + "diversity": { + "train": { + "mean_centered_corr": -0.019289543852210045, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04795046150684357 + }, + "val": { + "mean_centered_corr": -0.019224567338824272, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06529936194419861 + }, + "test": { + "mean_centered_corr": -0.01879088766872883, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08741392940282822 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.016868" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p001-evaluation/0.toml new file mode 100644 index 000000000..3961a64e2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p001-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..d6c1b1c10 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p001-evaluation/0/report.json @@ -0,0 +1,648 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2221833946825238, + "mae": 0.11801641434431076, + "r2": 0.6846178770065308, + "score": -0.2221833946825238 + }, + "val": { + "rmse": 0.23516710262665957, + "mae": 0.12856413424015045, + "r2": 0.6305935978889465, + "score": -0.23516710262665957 + }, + "test": { + "rmse": 0.2392893867678542, + "mae": 0.15676388144493103, + "r2": 0.6087210774421692, + "score": -0.2392893867678542 + } + }, + "time": "0:01:08.154288", + "diversity": { + "train": { + "mean_centered_corr": -0.019289543852210045, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04795046150684357 + }, + "val": { + "mean_centered_corr": -0.019224567338824272, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06529936194419861 + }, + "test": { + "mean_centered_corr": -0.01879088766872883, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08741392940282822 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..8aad1ae80 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22488731105453647, + "mae": 0.1187836155295372, + "r2": 0.6768949031829834, + "score": -0.22488731105453647 + }, + "val": { + "rmse": 0.23362806029940672, + "mae": 0.1260843127965927, + "r2": 0.6354129314422607, + "score": -0.23362806029940672 + }, + "test": { + "rmse": 0.23621330151542805, + "mae": 0.15148833394050598, + "r2": 0.6187162399291992, + "score": -0.23621330151542805 + } + }, + "time": "0:01:08.154288", + "diversity": { + "train": { + "mean_centered_corr": -0.019289543852210045, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04795046150684357 + }, + "val": { + "mean_centered_corr": -0.019224567338824272, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06529936194419861 + }, + "test": { + "mean_centered_corr": -0.01879088766872883, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08741392940282822 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 8, + 22, + 7, + 29, + 25, + 21 + ], + "head_selection_time": "0:00:00.120526" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..a0e3ff8d5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,652 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.23007499064462367, + "mae": 0.12374904006719589, + "r2": 0.6618162393569946, + "score": -0.23007499064462367 + }, + "val": { + "rmse": 0.2390328497980867, + "mae": 0.13070335984230042, + "r2": 0.6183489561080933, + "score": -0.2390328497980867 + }, + "test": { + "rmse": 0.24063334636332392, + "mae": 0.15138967335224152, + "r2": 0.6043134927749634, + "score": -0.24063334636332392 + } + }, + "time": "0:01:08.381106", + "diversity": { + "train": { + "mean_centered_corr": -0.018535790964961052, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048687759786844254 + }, + "val": { + "mean_centered_corr": -0.019753795117139816, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06561429798603058 + }, + "test": { + "mean_centered_corr": -0.01759418286383152, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09573547542095184 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21 + ], + "head_selection_time": "0:00:00.015250" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p005-evaluation/0.toml new file mode 100644 index 000000000..33808ab55 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p005-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..98b430c43 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p005-evaluation/0/report.json @@ -0,0 +1,648 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22270659663864684, + "mae": 0.11821237951517105, + "r2": 0.6831307411193848, + "score": -0.22270659663864684 + }, + "val": { + "rmse": 0.23541557264636437, + "mae": 0.1285467892885208, + "r2": 0.6298125982284546, + "score": -0.23541557264636437 + }, + "test": { + "rmse": 0.23955821427368978, + "mae": 0.15599055588245392, + "r2": 0.6078413724899292, + "score": -0.23955821427368978 + } + }, + "time": "0:01:08.381106", + "diversity": { + "train": { + "mean_centered_corr": -0.018535790964961052, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048687759786844254 + }, + "val": { + "mean_centered_corr": -0.019753795117139816, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06561429798603058 + }, + "test": { + "mean_centered_corr": -0.01759418286383152, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09573547542095184 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..738f85078 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,657 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2256070659700792, + "mae": 0.11890602111816406, + "r2": 0.6748234033584595, + "score": -0.2256070659700792 + }, + "val": { + "rmse": 0.2337388066681912, + "mae": 0.12727881968021393, + "r2": 0.6350672245025635, + "score": -0.2337388066681912 + }, + "test": { + "rmse": 0.2344355028261131, + "mae": 0.14499156177043915, + "r2": 0.6244339346885681, + "score": -0.2344355028261131 + } + }, + "time": "0:01:08.381106", + "diversity": { + "train": { + "mean_centered_corr": -0.018535790964961052, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048687759786844254 + }, + "val": { + "mean_centered_corr": -0.019753795117139816, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06561429798603058 + }, + "test": { + "mean_centered_corr": -0.01759418286383152, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09573547542095184 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21, + 29, + 8, + 0, + 19, + 22 + ], + "head_selection_time": "0:00:00.102535" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..21a562273 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,652 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2301618504668323, + "mae": 0.12545494735240936, + "r2": 0.6615608930587769, + "score": -0.2301618504668323 + }, + "val": { + "rmse": 0.23909857764435258, + "mae": 0.13463132083415985, + "r2": 0.6181390285491943, + "score": -0.23909857764435258 + }, + "test": { + "rmse": 0.24513721764174634, + "mae": 0.16250674426555634, + "r2": 0.5893629789352417, + "score": -0.24513721764174634 + } + }, + "time": "0:01:09.433151", + "diversity": { + "train": { + "mean_centered_corr": -0.01836317405104637, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04760168120265007 + }, + "val": { + "mean_centered_corr": -0.01840268261730671, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06696823239326477 + }, + "test": { + "mean_centered_corr": -0.018157226964831352, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09846261888742447 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21 + ], + "head_selection_time": "0:00:00.016053" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p01-evaluation/0.toml new file mode 100644 index 000000000..d859343b1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p01-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..e5c7ea752 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p01-evaluation/0/report.json @@ -0,0 +1,648 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2241623618168725, + "mae": 0.11926797032356262, + "r2": 0.6789746880531311, + "score": -0.2241623618168725 + }, + "val": { + "rmse": 0.23516078988949235, + "mae": 0.12952424585819244, + "r2": 0.6306134462356567, + "score": -0.23516078988949235 + }, + "test": { + "rmse": 0.24165682950771084, + "mae": 0.16096629202365875, + "r2": 0.600940465927124, + "score": -0.24165682950771084 + } + }, + "time": "0:01:09.433151", + "diversity": { + "train": { + "mean_centered_corr": -0.01836317405104637, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04760168120265007 + }, + "val": { + "mean_centered_corr": -0.01840268261730671, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06696823239326477 + }, + "test": { + "mean_centered_corr": -0.018157226964831352, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09846261888742447 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..aa499482f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,661 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22445249469109585, + "mae": 0.11813493072986603, + "r2": 0.6781431436538696, + "score": -0.22445249469109585 + }, + "val": { + "rmse": 0.23356265910507878, + "mae": 0.1266428679227829, + "r2": 0.6356170177459717, + "score": -0.23356265910507878 + }, + "test": { + "rmse": 0.23721029814115205, + "mae": 0.1523510068655014, + "r2": 0.6154908537864685, + "score": -0.23721029814115205 + } + }, + "time": "0:01:09.433151", + "diversity": { + "train": { + "mean_centered_corr": -0.01836317405104637, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04760168120265007 + }, + "val": { + "mean_centered_corr": -0.01840268261730671, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06696823239326477 + }, + "test": { + "mean_centered_corr": -0.018157226964831352, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09846261888742447 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21, + 7, + 4, + 22, + 19, + 8, + 18, + 29, + 7, + 25 + ], + "head_selection_time": "0:00:00.165917" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p02-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p02-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p02-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p02-best-head-evaluation/0/report.json new file mode 100644 index 000000000..da5a2ec8a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p02-best-head-evaluation/0/report.json @@ -0,0 +1,652 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23893091875282987, + "mae": 0.12647393345832825, + "r2": 0.6352808475494385, + "score": -0.23893091875282987 + }, + "val": { + "rmse": 0.23785345573254446, + "mae": 0.13102787733078003, + "r2": 0.6221058368682861, + "score": -0.23785345573254446 + }, + "test": { + "rmse": 0.23747813004889556, + "mae": 0.14464227855205536, + "r2": 0.6146221160888672, + "score": -0.23747813004889556 + } + }, + "time": "0:01:03.669566", + "diversity": { + "train": { + "mean_centered_corr": -0.019388988614082336, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04785213619470596 + }, + "val": { + "mean_centered_corr": -0.01973061077296734, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06109660863876343 + }, + "test": { + "mean_centered_corr": -0.019816331565380096, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08115967363119125 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 29 + ], + "head_selection_time": "0:00:00.015246" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p02-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p02-evaluation/0.toml new file mode 100644 index 000000000..931d62862 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p02-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.02 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p02-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p02-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p02-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p02-evaluation/0/report.json new file mode 100644 index 000000000..9292482c6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p02-evaluation/0/report.json @@ -0,0 +1,648 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23258982929101366, + "mae": 0.1219717338681221, + "r2": 0.6543828248977661, + "score": -0.23258982929101366 + }, + "val": { + "rmse": 0.23537261362228468, + "mae": 0.12668848037719727, + "r2": 0.6299476623535156, + "score": -0.23537261362228468 + }, + "test": { + "rmse": 0.23578993148696667, + "mae": 0.14991270005702972, + "r2": 0.6200817823410034, + "score": -0.23578993148696667 + } + }, + "time": "0:01:03.669566", + "diversity": { + "train": { + "mean_centered_corr": -0.019388988614082336, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04785213619470596 + }, + "val": { + "mean_centered_corr": -0.01973061077296734, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06109660863876343 + }, + "test": { + "mean_centered_corr": -0.019816331565380096, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08115967363119125 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p02-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p02-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p02-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p02-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..2cdf220da --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p02-greedy-heads-evaluation/0/report.json @@ -0,0 +1,661 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2347629753255239, + "mae": 0.12291525304317474, + "r2": 0.6478942632675171, + "score": -0.2347629753255239 + }, + "val": { + "rmse": 0.23340410361335404, + "mae": 0.12670208513736725, + "r2": 0.6361116170883179, + "score": -0.23340410361335404 + }, + "test": { + "rmse": 0.2382911617531396, + "mae": 0.15507616102695465, + "r2": 0.6119787693023682, + "score": -0.2382911617531396 + } + }, + "time": "0:01:03.669566", + "diversity": { + "train": { + "mean_centered_corr": -0.019388988614082336, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04785213619470596 + }, + "val": { + "mean_centered_corr": -0.01973061077296734, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06109660863876343 + }, + "test": { + "mean_centered_corr": -0.019816331565380096, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08115967363119125 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 29, + 25, + 28, + 0, + 16, + 21, + 4, + 25, + 1, + 6 + ], + "head_selection_time": "0:00:00.159810" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..dd025feaf --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,652 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2282046786382406, + "mae": 0.12737023830413818, + "r2": 0.6672922372817993, + "score": -0.2282046786382406 + }, + "val": { + "rmse": 0.2390544572113064, + "mae": 0.14095881581306458, + "r2": 0.6182799935340881, + "score": -0.2390544572113064 + }, + "test": { + "rmse": 0.25549556988031324, + "mae": 0.17997059226036072, + "r2": 0.5539265871047974, + "score": -0.25549556988031324 + } + }, + "time": "0:01:08.875317", + "diversity": { + "train": { + "mean_centered_corr": -0.019289543852210045, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04795046150684357 + }, + "val": { + "mean_centered_corr": -0.019224567338824272, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06529936194419861 + }, + "test": { + "mean_centered_corr": -0.01879088766872883, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08741392940282822 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015188" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p001-evaluation/0.toml new file mode 100644 index 000000000..4f90bab1a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p001-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..701ca6f99 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p001-evaluation/0/report.json @@ -0,0 +1,648 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2221833946825238, + "mae": 0.11801641434431076, + "r2": 0.6846178770065308, + "score": -0.2221833946825238 + }, + "val": { + "rmse": 0.23516710262665957, + "mae": 0.12856413424015045, + "r2": 0.6305935978889465, + "score": -0.23516710262665957 + }, + "test": { + "rmse": 0.2392893867678542, + "mae": 0.15676388144493103, + "r2": 0.6087210774421692, + "score": -0.2392893867678542 + } + }, + "time": "0:01:08.875317", + "diversity": { + "train": { + "mean_centered_corr": -0.019289543852210045, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04795046150684357 + }, + "val": { + "mean_centered_corr": -0.019224567338824272, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06529936194419861 + }, + "test": { + "mean_centered_corr": -0.01879088766872883, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08741392940282822 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..571745d70 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22488731105453647, + "mae": 0.1187836155295372, + "r2": 0.6768949031829834, + "score": -0.22488731105453647 + }, + "val": { + "rmse": 0.23362806029940672, + "mae": 0.1260843127965927, + "r2": 0.6354129314422607, + "score": -0.23362806029940672 + }, + "test": { + "rmse": 0.23621330151542805, + "mae": 0.15148833394050598, + "r2": 0.6187162399291992, + "score": -0.23621330151542805 + } + }, + "time": "0:01:08.875317", + "diversity": { + "train": { + "mean_centered_corr": -0.019289543852210045, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04795046150684357 + }, + "val": { + "mean_centered_corr": -0.019224567338824272, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06529936194419861 + }, + "test": { + "mean_centered_corr": -0.01879088766872883, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08741392940282822 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 8, + 22, + 7, + 29, + 25, + 21 + ], + "head_selection_time": "0:00:00.116284" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..09591f06c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,652 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.23007499064462367, + "mae": 0.12374904006719589, + "r2": 0.6618162393569946, + "score": -0.23007499064462367 + }, + "val": { + "rmse": 0.2390328497980867, + "mae": 0.13070335984230042, + "r2": 0.6183489561080933, + "score": -0.2390328497980867 + }, + "test": { + "rmse": 0.24063334636332392, + "mae": 0.15138967335224152, + "r2": 0.6043134927749634, + "score": -0.24063334636332392 + } + }, + "time": "0:01:09.803835", + "diversity": { + "train": { + "mean_centered_corr": -0.018535790964961052, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048687759786844254 + }, + "val": { + "mean_centered_corr": -0.019753795117139816, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06561429798603058 + }, + "test": { + "mean_centered_corr": -0.01759418286383152, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09573547542095184 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21 + ], + "head_selection_time": "0:00:00.015082" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p005-evaluation/0.toml new file mode 100644 index 000000000..3f8f8ebf0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p005-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..6f0d01a4d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p005-evaluation/0/report.json @@ -0,0 +1,648 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22270659663864684, + "mae": 0.11821237951517105, + "r2": 0.6831307411193848, + "score": -0.22270659663864684 + }, + "val": { + "rmse": 0.23541557264636437, + "mae": 0.1285467892885208, + "r2": 0.6298125982284546, + "score": -0.23541557264636437 + }, + "test": { + "rmse": 0.23955821427368978, + "mae": 0.15599055588245392, + "r2": 0.6078413724899292, + "score": -0.23955821427368978 + } + }, + "time": "0:01:09.803835", + "diversity": { + "train": { + "mean_centered_corr": -0.018535790964961052, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048687759786844254 + }, + "val": { + "mean_centered_corr": -0.019753795117139816, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06561429798603058 + }, + "test": { + "mean_centered_corr": -0.01759418286383152, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09573547542095184 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..4f5ce1df3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,657 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2256070659700792, + "mae": 0.11890602111816406, + "r2": 0.6748234033584595, + "score": -0.2256070659700792 + }, + "val": { + "rmse": 0.2337388066681912, + "mae": 0.12727881968021393, + "r2": 0.6350672245025635, + "score": -0.2337388066681912 + }, + "test": { + "rmse": 0.2344355028261131, + "mae": 0.14499156177043915, + "r2": 0.6244339346885681, + "score": -0.2344355028261131 + } + }, + "time": "0:01:09.803835", + "diversity": { + "train": { + "mean_centered_corr": -0.018535790964961052, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048687759786844254 + }, + "val": { + "mean_centered_corr": -0.019753795117139816, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06561429798603058 + }, + "test": { + "mean_centered_corr": -0.01759418286383152, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09573547542095184 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21, + 29, + 8, + 0, + 19, + 22 + ], + "head_selection_time": "0:00:00.102490" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..444867c42 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,652 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2301618504668323, + "mae": 0.12545494735240936, + "r2": 0.6615608930587769, + "score": -0.2301618504668323 + }, + "val": { + "rmse": 0.23909857764435258, + "mae": 0.13463132083415985, + "r2": 0.6181390285491943, + "score": -0.23909857764435258 + }, + "test": { + "rmse": 0.24513721764174634, + "mae": 0.16250674426555634, + "r2": 0.5893629789352417, + "score": -0.24513721764174634 + } + }, + "time": "0:01:09.043210", + "diversity": { + "train": { + "mean_centered_corr": -0.01836317405104637, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04760168120265007 + }, + "val": { + "mean_centered_corr": -0.01840268261730671, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06696823239326477 + }, + "test": { + "mean_centered_corr": -0.018157226964831352, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09846261888742447 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21 + ], + "head_selection_time": "0:00:00.015301" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p01-evaluation/0.toml new file mode 100644 index 000000000..8733113d2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p01-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..4b3d0d68d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p01-evaluation/0/report.json @@ -0,0 +1,648 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2241623618168725, + "mae": 0.11926797032356262, + "r2": 0.6789746880531311, + "score": -0.2241623618168725 + }, + "val": { + "rmse": 0.23516078988949235, + "mae": 0.12952424585819244, + "r2": 0.6306134462356567, + "score": -0.23516078988949235 + }, + "test": { + "rmse": 0.24165682950771084, + "mae": 0.16096629202365875, + "r2": 0.600940465927124, + "score": -0.24165682950771084 + } + }, + "time": "0:01:09.043210", + "diversity": { + "train": { + "mean_centered_corr": -0.01836317405104637, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04760168120265007 + }, + "val": { + "mean_centered_corr": -0.01840268261730671, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06696823239326477 + }, + "test": { + "mean_centered_corr": -0.018157226964831352, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09846261888742447 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..8f4003f0b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,661 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22445249469109585, + "mae": 0.11813493072986603, + "r2": 0.6781431436538696, + "score": -0.22445249469109585 + }, + "val": { + "rmse": 0.23356265910507878, + "mae": 0.1266428679227829, + "r2": 0.6356170177459717, + "score": -0.23356265910507878 + }, + "test": { + "rmse": 0.23721029814115205, + "mae": 0.1523510068655014, + "r2": 0.6154908537864685, + "score": -0.23721029814115205 + } + }, + "time": "0:01:09.043210", + "diversity": { + "train": { + "mean_centered_corr": -0.01836317405104637, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04760168120265007 + }, + "val": { + "mean_centered_corr": -0.01840268261730671, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06696823239326477 + }, + "test": { + "mean_centered_corr": -0.018157226964831352, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09846261888742447 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21, + 7, + 4, + 22, + 19, + 8, + 18, + 29, + 7, + 25 + ], + "head_selection_time": "0:00:00.167664" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p02-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p02-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p02-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p02-best-head-evaluation/0/report.json new file mode 100644 index 000000000..e01bf8f42 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p02-best-head-evaluation/0/report.json @@ -0,0 +1,652 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23893091875282987, + "mae": 0.12647393345832825, + "r2": 0.6352808475494385, + "score": -0.23893091875282987 + }, + "val": { + "rmse": 0.23785345573254446, + "mae": 0.13102787733078003, + "r2": 0.6221058368682861, + "score": -0.23785345573254446 + }, + "test": { + "rmse": 0.23747813004889556, + "mae": 0.14464227855205536, + "r2": 0.6146221160888672, + "score": -0.23747813004889556 + } + }, + "time": "0:01:01.731990", + "diversity": { + "train": { + "mean_centered_corr": -0.019388988614082336, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04785213619470596 + }, + "val": { + "mean_centered_corr": -0.01973061077296734, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06109660863876343 + }, + "test": { + "mean_centered_corr": -0.019816331565380096, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08115967363119125 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 29 + ], + "head_selection_time": "0:00:00.015297" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p02-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p02-evaluation/0.toml new file mode 100644 index 000000000..d87358e44 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p02-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.02 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p02-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p02-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p02-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p02-evaluation/0/report.json new file mode 100644 index 000000000..ba8bcae09 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p02-evaluation/0/report.json @@ -0,0 +1,648 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23258982929101366, + "mae": 0.1219717338681221, + "r2": 0.6543828248977661, + "score": -0.23258982929101366 + }, + "val": { + "rmse": 0.23537261362228468, + "mae": 0.12668848037719727, + "r2": 0.6299476623535156, + "score": -0.23537261362228468 + }, + "test": { + "rmse": 0.23578993148696667, + "mae": 0.14991270005702972, + "r2": 0.6200817823410034, + "score": -0.23578993148696667 + } + }, + "time": "0:01:01.731990", + "diversity": { + "train": { + "mean_centered_corr": -0.019388988614082336, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04785213619470596 + }, + "val": { + "mean_centered_corr": -0.01973061077296734, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06109660863876343 + }, + "test": { + "mean_centered_corr": -0.019816331565380096, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08115967363119125 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p02-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p02-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p02-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p02-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..3d0694006 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p02-greedy-heads-evaluation/0/report.json @@ -0,0 +1,661 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2347629753255239, + "mae": 0.12291525304317474, + "r2": 0.6478942632675171, + "score": -0.2347629753255239 + }, + "val": { + "rmse": 0.23340410361335404, + "mae": 0.12670208513736725, + "r2": 0.6361116170883179, + "score": -0.23340410361335404 + }, + "test": { + "rmse": 0.2382911617531396, + "mae": 0.15507616102695465, + "r2": 0.6119787693023682, + "score": -0.2382911617531396 + } + }, + "time": "0:01:01.731990", + "diversity": { + "train": { + "mean_centered_corr": -0.019388988614082336, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04785213619470596 + }, + "val": { + "mean_centered_corr": -0.01973061077296734, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06109660863876343 + }, + "test": { + "mean_centered_corr": -0.019816331565380096, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08115967363119125 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 29, + 25, + 28, + 0, + 16, + 21, + 4, + 25, + 1, + 6 + ], + "head_selection_time": "0:00:00.165656" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..02606d789 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,652 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2282046786382406, + "mae": 0.12737023830413818, + "r2": 0.6672922372817993, + "score": -0.2282046786382406 + }, + "val": { + "rmse": 0.2390544572113064, + "mae": 0.14095881581306458, + "r2": 0.6182799935340881, + "score": -0.2390544572113064 + }, + "test": { + "rmse": 0.25549556988031324, + "mae": 0.17997059226036072, + "r2": 0.5539265871047974, + "score": -0.25549556988031324 + } + }, + "time": "0:01:06.666597", + "diversity": { + "train": { + "mean_centered_corr": -0.019289543852210045, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04795046150684357 + }, + "val": { + "mean_centered_corr": -0.019224567338824272, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06529936194419861 + }, + "test": { + "mean_centered_corr": -0.01879088766872883, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08741392940282822 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015150" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p001-evaluation/0.toml new file mode 100644 index 000000000..97bb8b676 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p001-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..3ffd171d4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p001-evaluation/0/report.json @@ -0,0 +1,648 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2221833946825238, + "mae": 0.11801641434431076, + "r2": 0.6846178770065308, + "score": -0.2221833946825238 + }, + "val": { + "rmse": 0.23516710262665957, + "mae": 0.12856413424015045, + "r2": 0.6305935978889465, + "score": -0.23516710262665957 + }, + "test": { + "rmse": 0.2392893867678542, + "mae": 0.15676388144493103, + "r2": 0.6087210774421692, + "score": -0.2392893867678542 + } + }, + "time": "0:01:06.666597", + "diversity": { + "train": { + "mean_centered_corr": -0.019289543852210045, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04795046150684357 + }, + "val": { + "mean_centered_corr": -0.019224567338824272, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06529936194419861 + }, + "test": { + "mean_centered_corr": -0.01879088766872883, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08741392940282822 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..d2bc4db76 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22488731105453647, + "mae": 0.1187836155295372, + "r2": 0.6768949031829834, + "score": -0.22488731105453647 + }, + "val": { + "rmse": 0.23362806029940672, + "mae": 0.1260843127965927, + "r2": 0.6354129314422607, + "score": -0.23362806029940672 + }, + "test": { + "rmse": 0.23621330151542805, + "mae": 0.15148833394050598, + "r2": 0.6187162399291992, + "score": -0.23621330151542805 + } + }, + "time": "0:01:06.666597", + "diversity": { + "train": { + "mean_centered_corr": -0.019289543852210045, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04795046150684357 + }, + "val": { + "mean_centered_corr": -0.019224567338824272, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06529936194419861 + }, + "test": { + "mean_centered_corr": -0.01879088766872883, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08741392940282822 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 8, + 22, + 7, + 29, + 25, + 21 + ], + "head_selection_time": "0:00:00.116357" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..e2a883962 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,652 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.23007499064462367, + "mae": 0.12374904006719589, + "r2": 0.6618162393569946, + "score": -0.23007499064462367 + }, + "val": { + "rmse": 0.2390328497980867, + "mae": 0.13070335984230042, + "r2": 0.6183489561080933, + "score": -0.2390328497980867 + }, + "test": { + "rmse": 0.24063334636332392, + "mae": 0.15138967335224152, + "r2": 0.6043134927749634, + "score": -0.24063334636332392 + } + }, + "time": "0:01:08.046540", + "diversity": { + "train": { + "mean_centered_corr": -0.018535790964961052, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048687759786844254 + }, + "val": { + "mean_centered_corr": -0.019753795117139816, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06561429798603058 + }, + "test": { + "mean_centered_corr": -0.01759418286383152, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09573547542095184 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21 + ], + "head_selection_time": "0:00:00.015673" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p005-evaluation/0.toml new file mode 100644 index 000000000..af87226ff --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p005-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..b67ad5b90 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p005-evaluation/0/report.json @@ -0,0 +1,648 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22270659663864684, + "mae": 0.11821237951517105, + "r2": 0.6831307411193848, + "score": -0.22270659663864684 + }, + "val": { + "rmse": 0.23541557264636437, + "mae": 0.1285467892885208, + "r2": 0.6298125982284546, + "score": -0.23541557264636437 + }, + "test": { + "rmse": 0.23955821427368978, + "mae": 0.15599055588245392, + "r2": 0.6078413724899292, + "score": -0.23955821427368978 + } + }, + "time": "0:01:08.046540", + "diversity": { + "train": { + "mean_centered_corr": -0.018535790964961052, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048687759786844254 + }, + "val": { + "mean_centered_corr": -0.019753795117139816, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06561429798603058 + }, + "test": { + "mean_centered_corr": -0.01759418286383152, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09573547542095184 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..c9e5cee60 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,657 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2256070659700792, + "mae": 0.11890602111816406, + "r2": 0.6748234033584595, + "score": -0.2256070659700792 + }, + "val": { + "rmse": 0.2337388066681912, + "mae": 0.12727881968021393, + "r2": 0.6350672245025635, + "score": -0.2337388066681912 + }, + "test": { + "rmse": 0.2344355028261131, + "mae": 0.14499156177043915, + "r2": 0.6244339346885681, + "score": -0.2344355028261131 + } + }, + "time": "0:01:08.046540", + "diversity": { + "train": { + "mean_centered_corr": -0.018535790964961052, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048687759786844254 + }, + "val": { + "mean_centered_corr": -0.019753795117139816, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06561429798603058 + }, + "test": { + "mean_centered_corr": -0.01759418286383152, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09573547542095184 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21, + 29, + 8, + 0, + 19, + 22 + ], + "head_selection_time": "0:00:00.104607" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..82c6a3181 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,652 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2301618504668323, + "mae": 0.12545494735240936, + "r2": 0.6615608930587769, + "score": -0.2301618504668323 + }, + "val": { + "rmse": 0.23909857764435258, + "mae": 0.13463132083415985, + "r2": 0.6181390285491943, + "score": -0.23909857764435258 + }, + "test": { + "rmse": 0.24513721764174634, + "mae": 0.16250674426555634, + "r2": 0.5893629789352417, + "score": -0.24513721764174634 + } + }, + "time": "0:01:07.636035", + "diversity": { + "train": { + "mean_centered_corr": -0.01836317405104637, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04760168120265007 + }, + "val": { + "mean_centered_corr": -0.01840268261730671, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06696823239326477 + }, + "test": { + "mean_centered_corr": -0.018157226964831352, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09846261888742447 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21 + ], + "head_selection_time": "0:00:00.015220" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p01-evaluation/0.toml new file mode 100644 index 000000000..46371ee6c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p01-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..26e59da91 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p01-evaluation/0/report.json @@ -0,0 +1,648 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2241623618168725, + "mae": 0.11926797032356262, + "r2": 0.6789746880531311, + "score": -0.2241623618168725 + }, + "val": { + "rmse": 0.23516078988949235, + "mae": 0.12952424585819244, + "r2": 0.6306134462356567, + "score": -0.23516078988949235 + }, + "test": { + "rmse": 0.24165682950771084, + "mae": 0.16096629202365875, + "r2": 0.600940465927124, + "score": -0.24165682950771084 + } + }, + "time": "0:01:07.636035", + "diversity": { + "train": { + "mean_centered_corr": -0.01836317405104637, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04760168120265007 + }, + "val": { + "mean_centered_corr": -0.01840268261730671, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06696823239326477 + }, + "test": { + "mean_centered_corr": -0.018157226964831352, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09846261888742447 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..7140eede2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,661 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22445249469109585, + "mae": 0.11813493072986603, + "r2": 0.6781431436538696, + "score": -0.22445249469109585 + }, + "val": { + "rmse": 0.23356265910507878, + "mae": 0.1266428679227829, + "r2": 0.6356170177459717, + "score": -0.23356265910507878 + }, + "test": { + "rmse": 0.23721029814115205, + "mae": 0.1523510068655014, + "r2": 0.6154908537864685, + "score": -0.23721029814115205 + } + }, + "time": "0:01:07.636035", + "diversity": { + "train": { + "mean_centered_corr": -0.01836317405104637, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04760168120265007 + }, + "val": { + "mean_centered_corr": -0.01840268261730671, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06696823239326477 + }, + "test": { + "mean_centered_corr": -0.018157226964831352, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09846261888742447 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21, + 7, + 4, + 22, + 19, + 8, + 18, + 29, + 7, + 25 + ], + "head_selection_time": "0:00:00.161310" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p02-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p02-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p02-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p02-best-head-evaluation/0/report.json new file mode 100644 index 000000000..3e9d2582e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p02-best-head-evaluation/0/report.json @@ -0,0 +1,652 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23893091875282987, + "mae": 0.12647393345832825, + "r2": 0.6352808475494385, + "score": -0.23893091875282987 + }, + "val": { + "rmse": 0.23785345573254446, + "mae": 0.13102787733078003, + "r2": 0.6221058368682861, + "score": -0.23785345573254446 + }, + "test": { + "rmse": 0.23747813004889556, + "mae": 0.14464227855205536, + "r2": 0.6146221160888672, + "score": -0.23747813004889556 + } + }, + "time": "0:01:03.090464", + "diversity": { + "train": { + "mean_centered_corr": -0.019388988614082336, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04785213619470596 + }, + "val": { + "mean_centered_corr": -0.01973061077296734, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06109660863876343 + }, + "test": { + "mean_centered_corr": -0.019816331565380096, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08115967363119125 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 29 + ], + "head_selection_time": "0:00:00.015272" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p02-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p02-evaluation/0.toml new file mode 100644 index 000000000..0e85933ea --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p02-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.02 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p02-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p02-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p02-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p02-evaluation/0/report.json new file mode 100644 index 000000000..f32ddca60 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p02-evaluation/0/report.json @@ -0,0 +1,648 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23258982929101366, + "mae": 0.1219717338681221, + "r2": 0.6543828248977661, + "score": -0.23258982929101366 + }, + "val": { + "rmse": 0.23537261362228468, + "mae": 0.12668848037719727, + "r2": 0.6299476623535156, + "score": -0.23537261362228468 + }, + "test": { + "rmse": 0.23578993148696667, + "mae": 0.14991270005702972, + "r2": 0.6200817823410034, + "score": -0.23578993148696667 + } + }, + "time": "0:01:03.090464", + "diversity": { + "train": { + "mean_centered_corr": -0.019388988614082336, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04785213619470596 + }, + "val": { + "mean_centered_corr": -0.01973061077296734, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06109660863876343 + }, + "test": { + "mean_centered_corr": -0.019816331565380096, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08115967363119125 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p02-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p02-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p02-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p02-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..3e54a594f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p02-greedy-heads-evaluation/0/report.json @@ -0,0 +1,661 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2347629753255239, + "mae": 0.12291525304317474, + "r2": 0.6478942632675171, + "score": -0.2347629753255239 + }, + "val": { + "rmse": 0.23340410361335404, + "mae": 0.12670208513736725, + "r2": 0.6361116170883179, + "score": -0.23340410361335404 + }, + "test": { + "rmse": 0.2382911617531396, + "mae": 0.15507616102695465, + "r2": 0.6119787693023682, + "score": -0.2382911617531396 + } + }, + "time": "0:01:03.090464", + "diversity": { + "train": { + "mean_centered_corr": -0.019388988614082336, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04785213619470596 + }, + "val": { + "mean_centered_corr": -0.01973061077296734, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06109660863876343 + }, + "test": { + "mean_centered_corr": -0.019816331565380096, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08115967363119125 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 29, + 25, + 28, + 0, + 16, + 21, + 4, + 25, + 1, + 6 + ], + "head_selection_time": "0:00:00.158927" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..1a53b3e6a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,652 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2282046786382406, + "mae": 0.12737023830413818, + "r2": 0.6672922372817993, + "score": -0.2282046786382406 + }, + "val": { + "rmse": 0.2390544572113064, + "mae": 0.14095881581306458, + "r2": 0.6182799935340881, + "score": -0.2390544572113064 + }, + "test": { + "rmse": 0.25549556988031324, + "mae": 0.17997059226036072, + "r2": 0.5539265871047974, + "score": -0.25549556988031324 + } + }, + "time": "0:01:08.087798", + "diversity": { + "train": { + "mean_centered_corr": -0.019289543852210045, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04795046150684357 + }, + "val": { + "mean_centered_corr": -0.019224567338824272, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06529936194419861 + }, + "test": { + "mean_centered_corr": -0.01879088766872883, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08741392940282822 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015177" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p001-evaluation/0.toml new file mode 100644 index 000000000..92f3cea1e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p001-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..e01a1f584 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p001-evaluation/0/report.json @@ -0,0 +1,648 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2221833946825238, + "mae": 0.11801641434431076, + "r2": 0.6846178770065308, + "score": -0.2221833946825238 + }, + "val": { + "rmse": 0.23516710262665957, + "mae": 0.12856413424015045, + "r2": 0.6305935978889465, + "score": -0.23516710262665957 + }, + "test": { + "rmse": 0.2392893867678542, + "mae": 0.15676388144493103, + "r2": 0.6087210774421692, + "score": -0.2392893867678542 + } + }, + "time": "0:01:08.087798", + "diversity": { + "train": { + "mean_centered_corr": -0.019289543852210045, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04795046150684357 + }, + "val": { + "mean_centered_corr": -0.019224567338824272, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06529936194419861 + }, + "test": { + "mean_centered_corr": -0.01879088766872883, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08741392940282822 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..485b0677c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22488731105453647, + "mae": 0.1187836155295372, + "r2": 0.6768949031829834, + "score": -0.22488731105453647 + }, + "val": { + "rmse": 0.23362806029940672, + "mae": 0.1260843127965927, + "r2": 0.6354129314422607, + "score": -0.23362806029940672 + }, + "test": { + "rmse": 0.23621330151542805, + "mae": 0.15148833394050598, + "r2": 0.6187162399291992, + "score": -0.23621330151542805 + } + }, + "time": "0:01:08.087798", + "diversity": { + "train": { + "mean_centered_corr": -0.019289543852210045, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04795046150684357 + }, + "val": { + "mean_centered_corr": -0.019224567338824272, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06529936194419861 + }, + "test": { + "mean_centered_corr": -0.01879088766872883, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08741392940282822 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 8, + 22, + 7, + 29, + 25, + 21 + ], + "head_selection_time": "0:00:00.115387" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..e6a148498 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,652 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.23007499064462367, + "mae": 0.12374904006719589, + "r2": 0.6618162393569946, + "score": -0.23007499064462367 + }, + "val": { + "rmse": 0.2390328497980867, + "mae": 0.13070335984230042, + "r2": 0.6183489561080933, + "score": -0.2390328497980867 + }, + "test": { + "rmse": 0.24063334636332392, + "mae": 0.15138967335224152, + "r2": 0.6043134927749634, + "score": -0.24063334636332392 + } + }, + "time": "0:01:06.267932", + "diversity": { + "train": { + "mean_centered_corr": -0.018535790964961052, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048687759786844254 + }, + "val": { + "mean_centered_corr": -0.019753795117139816, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06561429798603058 + }, + "test": { + "mean_centered_corr": -0.01759418286383152, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09573547542095184 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21 + ], + "head_selection_time": "0:00:00.015159" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p005-evaluation/0.toml new file mode 100644 index 000000000..82e1f7182 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p005-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..b8a5dad5e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p005-evaluation/0/report.json @@ -0,0 +1,648 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22270659663864684, + "mae": 0.11821237951517105, + "r2": 0.6831307411193848, + "score": -0.22270659663864684 + }, + "val": { + "rmse": 0.23541557264636437, + "mae": 0.1285467892885208, + "r2": 0.6298125982284546, + "score": -0.23541557264636437 + }, + "test": { + "rmse": 0.23955821427368978, + "mae": 0.15599055588245392, + "r2": 0.6078413724899292, + "score": -0.23955821427368978 + } + }, + "time": "0:01:06.267932", + "diversity": { + "train": { + "mean_centered_corr": -0.018535790964961052, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048687759786844254 + }, + "val": { + "mean_centered_corr": -0.019753795117139816, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06561429798603058 + }, + "test": { + "mean_centered_corr": -0.01759418286383152, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09573547542095184 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..0ed641c86 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,657 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2256070659700792, + "mae": 0.11890602111816406, + "r2": 0.6748234033584595, + "score": -0.2256070659700792 + }, + "val": { + "rmse": 0.2337388066681912, + "mae": 0.12727881968021393, + "r2": 0.6350672245025635, + "score": -0.2337388066681912 + }, + "test": { + "rmse": 0.2344355028261131, + "mae": 0.14499156177043915, + "r2": 0.6244339346885681, + "score": -0.2344355028261131 + } + }, + "time": "0:01:06.267932", + "diversity": { + "train": { + "mean_centered_corr": -0.018535790964961052, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048687759786844254 + }, + "val": { + "mean_centered_corr": -0.019753795117139816, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06561429798603058 + }, + "test": { + "mean_centered_corr": -0.01759418286383152, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09573547542095184 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21, + 29, + 8, + 0, + 19, + 22 + ], + "head_selection_time": "0:00:00.100716" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..a84a97944 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,652 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2301618504668323, + "mae": 0.12545494735240936, + "r2": 0.6615608930587769, + "score": -0.2301618504668323 + }, + "val": { + "rmse": 0.23909857764435258, + "mae": 0.13463132083415985, + "r2": 0.6181390285491943, + "score": -0.23909857764435258 + }, + "test": { + "rmse": 0.24513721764174634, + "mae": 0.16250674426555634, + "r2": 0.5893629789352417, + "score": -0.24513721764174634 + } + }, + "time": "0:01:06.451437", + "diversity": { + "train": { + "mean_centered_corr": -0.01836317405104637, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04760168120265007 + }, + "val": { + "mean_centered_corr": -0.01840268261730671, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06696823239326477 + }, + "test": { + "mean_centered_corr": -0.018157226964831352, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09846261888742447 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21 + ], + "head_selection_time": "0:00:00.014906" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p01-evaluation/0.toml new file mode 100644 index 000000000..11cc63c07 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p01-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..4d2447da1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p01-evaluation/0/report.json @@ -0,0 +1,648 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2241623618168725, + "mae": 0.11926797032356262, + "r2": 0.6789746880531311, + "score": -0.2241623618168725 + }, + "val": { + "rmse": 0.23516078988949235, + "mae": 0.12952424585819244, + "r2": 0.6306134462356567, + "score": -0.23516078988949235 + }, + "test": { + "rmse": 0.24165682950771084, + "mae": 0.16096629202365875, + "r2": 0.600940465927124, + "score": -0.24165682950771084 + } + }, + "time": "0:01:06.451437", + "diversity": { + "train": { + "mean_centered_corr": -0.01836317405104637, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04760168120265007 + }, + "val": { + "mean_centered_corr": -0.01840268261730671, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06696823239326477 + }, + "test": { + "mean_centered_corr": -0.018157226964831352, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09846261888742447 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..71f1b5335 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,661 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22445249469109585, + "mae": 0.11813493072986603, + "r2": 0.6781431436538696, + "score": -0.22445249469109585 + }, + "val": { + "rmse": 0.23356265910507878, + "mae": 0.1266428679227829, + "r2": 0.6356170177459717, + "score": -0.23356265910507878 + }, + "test": { + "rmse": 0.23721029814115205, + "mae": 0.1523510068655014, + "r2": 0.6154908537864685, + "score": -0.23721029814115205 + } + }, + "time": "0:01:06.451437", + "diversity": { + "train": { + "mean_centered_corr": -0.01836317405104637, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04760168120265007 + }, + "val": { + "mean_centered_corr": -0.01840268261730671, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06696823239326477 + }, + "test": { + "mean_centered_corr": -0.018157226964831352, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09846261888742447 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21, + 7, + 4, + 22, + 19, + 8, + 18, + 29, + 7, + 25 + ], + "head_selection_time": "0:00:00.157239" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p02-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p02-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p02-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p02-best-head-evaluation/0/report.json new file mode 100644 index 000000000..d60489982 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p02-best-head-evaluation/0/report.json @@ -0,0 +1,652 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23893091875282987, + "mae": 0.12647393345832825, + "r2": 0.6352808475494385, + "score": -0.23893091875282987 + }, + "val": { + "rmse": 0.23785345573254446, + "mae": 0.13102787733078003, + "r2": 0.6221058368682861, + "score": -0.23785345573254446 + }, + "test": { + "rmse": 0.23747813004889556, + "mae": 0.14464227855205536, + "r2": 0.6146221160888672, + "score": -0.23747813004889556 + } + }, + "time": "0:01:03.527749", + "diversity": { + "train": { + "mean_centered_corr": -0.019388988614082336, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04785213619470596 + }, + "val": { + "mean_centered_corr": -0.01973061077296734, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06109660863876343 + }, + "test": { + "mean_centered_corr": -0.019816331565380096, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08115967363119125 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 29 + ], + "head_selection_time": "0:00:00.015077" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p02-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p02-evaluation/0.toml new file mode 100644 index 000000000..cb443d190 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p02-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.02 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p02-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p02-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p02-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p02-evaluation/0/report.json new file mode 100644 index 000000000..a1773ffe5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p02-evaluation/0/report.json @@ -0,0 +1,648 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23258982929101366, + "mae": 0.1219717338681221, + "r2": 0.6543828248977661, + "score": -0.23258982929101366 + }, + "val": { + "rmse": 0.23537261362228468, + "mae": 0.12668848037719727, + "r2": 0.6299476623535156, + "score": -0.23537261362228468 + }, + "test": { + "rmse": 0.23578993148696667, + "mae": 0.14991270005702972, + "r2": 0.6200817823410034, + "score": -0.23578993148696667 + } + }, + "time": "0:01:03.527749", + "diversity": { + "train": { + "mean_centered_corr": -0.019388988614082336, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04785213619470596 + }, + "val": { + "mean_centered_corr": -0.01973061077296734, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06109660863876343 + }, + "test": { + "mean_centered_corr": -0.019816331565380096, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08115967363119125 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p02-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p02-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p02-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p02-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..1ff6458ff --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p02-greedy-heads-evaluation/0/report.json @@ -0,0 +1,661 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2347629753255239, + "mae": 0.12291525304317474, + "r2": 0.6478942632675171, + "score": -0.2347629753255239 + }, + "val": { + "rmse": 0.23340410361335404, + "mae": 0.12670208513736725, + "r2": 0.6361116170883179, + "score": -0.23340410361335404 + }, + "test": { + "rmse": 0.2382911617531396, + "mae": 0.15507616102695465, + "r2": 0.6119787693023682, + "score": -0.2382911617531396 + } + }, + "time": "0:01:03.527749", + "diversity": { + "train": { + "mean_centered_corr": -0.019388988614082336, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04785213619470596 + }, + "val": { + "mean_centered_corr": -0.01973061077296734, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06109660863876343 + }, + "test": { + "mean_centered_corr": -0.019816331565380096, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08115967363119125 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 29, + 25, + 28, + 0, + 16, + 21, + 4, + 25, + 1, + 6 + ], + "head_selection_time": "0:00:00.158406" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..82a0d9f14 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22820973096491176, + "mae": 0.12737685441970825, + "r2": 0.667277455329895, + "score": -0.22820973096491176 + }, + "val": { + "rmse": 0.23905700508983188, + "mae": 0.14095449447631836, + "r2": 0.6182718276977539, + "score": -0.23905700508983188 + }, + "test": { + "rmse": 0.2554892709638668, + "mae": 0.17996472120285034, + "r2": 0.5539485812187195, + "score": -0.2554892709638668 + } + }, + "time": "0:01:27.923469", + "diversity": { + "train": { + "mean_centered_corr": -0.019317513331770897, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04794894531369209 + }, + "val": { + "mean_centered_corr": -0.019240468740463257, + "mean_pairwise_disagreement": NaN, + "member_std": 0.065296970307827 + }, + "test": { + "mean_centered_corr": -0.018747208639979362, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08741389214992523 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.016127" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p001-evaluation/0.toml new file mode 100644 index 000000000..12f1a9ea9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p001-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..650fcbc1c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p001-evaluation/0/report.json @@ -0,0 +1,654 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22218533123207995, + "mae": 0.11801747977733612, + "r2": 0.6846123933792114, + "score": -0.22218533123207995 + }, + "val": { + "rmse": 0.23516964509902982, + "mae": 0.1285666972398758, + "r2": 0.6305856704711914, + "score": -0.23516964509902982 + }, + "test": { + "rmse": 0.23928895864366434, + "mae": 0.15676262974739075, + "r2": 0.608722448348999, + "score": -0.23928895864366434 + } + }, + "time": "0:01:27.923469", + "diversity": { + "train": { + "mean_centered_corr": -0.019317513331770897, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04794894531369209 + }, + "val": { + "mean_centered_corr": -0.019240468740463257, + "mean_pairwise_disagreement": NaN, + "member_std": 0.065296970307827 + }, + "test": { + "mean_centered_corr": -0.018747208639979362, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08741389214992523 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..3f00e7ab6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,664 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22489126180626096, + "mae": 0.11878597736358643, + "r2": 0.6768835783004761, + "score": -0.22489126180626096 + }, + "val": { + "rmse": 0.23363252496566134, + "mae": 0.12608498334884644, + "r2": 0.6353989839553833, + "score": -0.23363252496566134 + }, + "test": { + "rmse": 0.2362188764537236, + "mae": 0.15149326622486115, + "r2": 0.618698239326477, + "score": -0.2362188764537236 + } + }, + "time": "0:01:27.923469", + "diversity": { + "train": { + "mean_centered_corr": -0.019317513331770897, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04794894531369209 + }, + "val": { + "mean_centered_corr": -0.019240468740463257, + "mean_pairwise_disagreement": NaN, + "member_std": 0.065296970307827 + }, + "test": { + "mean_centered_corr": -0.018747208639979362, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08741389214992523 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 8, + 22, + 7, + 29, + 25, + 21 + ], + "head_selection_time": "0:00:00.117264" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..860fa6472 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.23007882803014523, + "mae": 0.12374971061944962, + "r2": 0.6618049740791321, + "score": -0.23007882803014523 + }, + "val": { + "rmse": 0.23901989843968774, + "mae": 0.13069558143615723, + "r2": 0.6183903217315674, + "score": -0.23901989843968774 + }, + "test": { + "rmse": 0.24063495640158664, + "mae": 0.1513846516609192, + "r2": 0.6043082475662231, + "score": -0.24063495640158664 + } + }, + "time": "0:01:28.376248", + "diversity": { + "train": { + "mean_centered_corr": -0.018541699275374413, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04868823289871216 + }, + "val": { + "mean_centered_corr": -0.019729774445295334, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06561318784952164 + }, + "test": { + "mean_centered_corr": -0.01752825267612934, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09573257714509964 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21 + ], + "head_selection_time": "0:00:00.015332" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p005-evaluation/0.toml new file mode 100644 index 000000000..ae4744a5f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p005-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..0b121938f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p005-evaluation/0/report.json @@ -0,0 +1,654 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22270596936228843, + "mae": 0.11821265518665314, + "r2": 0.6831325888633728, + "score": -0.22270596936228843 + }, + "val": { + "rmse": 0.23541076200529618, + "mae": 0.1285439282655716, + "r2": 0.6298277378082275, + "score": -0.23541076200529618 + }, + "test": { + "rmse": 0.2395584319829453, + "mae": 0.15599201619625092, + "r2": 0.6078406572341919, + "score": -0.2395584319829453 + } + }, + "time": "0:01:28.376248", + "diversity": { + "train": { + "mean_centered_corr": -0.018541699275374413, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04868823289871216 + }, + "val": { + "mean_centered_corr": -0.019729774445295334, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06561318784952164 + }, + "test": { + "mean_centered_corr": -0.01752825267612934, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09573257714509964 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..5936c02a0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,663 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22560705771393183, + "mae": 0.11890538036823273, + "r2": 0.6748234033584595, + "score": -0.22560705771393183 + }, + "val": { + "rmse": 0.23373660723675202, + "mae": 0.1272813230752945, + "r2": 0.6350740790367126, + "score": -0.23373660723675202 + }, + "test": { + "rmse": 0.23443243594536417, + "mae": 0.14499256014823914, + "r2": 0.624443769454956, + "score": -0.23443243594536417 + } + }, + "time": "0:01:28.376248", + "diversity": { + "train": { + "mean_centered_corr": -0.018541699275374413, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04868823289871216 + }, + "val": { + "mean_centered_corr": -0.019729774445295334, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06561318784952164 + }, + "test": { + "mean_centered_corr": -0.01752825267612934, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09573257714509964 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21, + 29, + 8, + 0, + 19, + 22 + ], + "head_selection_time": "0:00:00.101988" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..3d02dfe64 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.23015652536762907, + "mae": 0.12545116245746613, + "r2": 0.661576509475708, + "score": -0.23015652536762907 + }, + "val": { + "rmse": 0.239085684381371, + "mae": 0.13462616503238678, + "r2": 0.6181802749633789, + "score": -0.239085684381371 + }, + "test": { + "rmse": 0.24513387433279307, + "mae": 0.16250139474868774, + "r2": 0.5893741846084595, + "score": -0.24513387433279307 + } + }, + "time": "0:01:28.822139", + "diversity": { + "train": { + "mean_centered_corr": -0.01839604787528515, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0476016141474247 + }, + "val": { + "mean_centered_corr": -0.0184602253139019, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06696488708257675 + }, + "test": { + "mean_centered_corr": -0.018153522163629532, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09845950454473495 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21 + ], + "head_selection_time": "0:00:00.015540" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p01-evaluation/0.toml new file mode 100644 index 000000000..2bf833500 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p01-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..f30d39703 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p01-evaluation/0/report.json @@ -0,0 +1,654 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22416379102176556, + "mae": 0.11926887929439545, + "r2": 0.6789705753326416, + "score": -0.22416379102176556 + }, + "val": { + "rmse": 0.23516104335271928, + "mae": 0.12952451407909393, + "r2": 0.6306126713752747, + "score": -0.23516104335271928 + }, + "test": { + "rmse": 0.24165658285763553, + "mae": 0.1609659045934677, + "r2": 0.6009412407875061, + "score": -0.24165658285763553 + } + }, + "time": "0:01:28.822139", + "diversity": { + "train": { + "mean_centered_corr": -0.01839604787528515, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0476016141474247 + }, + "val": { + "mean_centered_corr": -0.0184602253139019, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06696488708257675 + }, + "test": { + "mean_centered_corr": -0.018153522163629532, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09845950454473495 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..b2716d4c8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,667 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22445475190340167, + "mae": 0.11813689768314362, + "r2": 0.6781366467475891, + "score": -0.22445475190340167 + }, + "val": { + "rmse": 0.23356035433981295, + "mae": 0.1266433298587799, + "r2": 0.6356241703033447, + "score": -0.23356035433981295 + }, + "test": { + "rmse": 0.2372102431750833, + "mae": 0.15235204994678497, + "r2": 0.6154910326004028, + "score": -0.2372102431750833 + } + }, + "time": "0:01:28.822139", + "diversity": { + "train": { + "mean_centered_corr": -0.01839604787528515, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0476016141474247 + }, + "val": { + "mean_centered_corr": -0.0184602253139019, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06696488708257675 + }, + "test": { + "mean_centered_corr": -0.018153522163629532, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09845950454473495 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21, + 7, + 4, + 22, + 19, + 8, + 18, + 7, + 29, + 25 + ], + "head_selection_time": "0:00:00.163829" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p02-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p02-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p02-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p02-best-head-evaluation/0/report.json new file mode 100644 index 000000000..17b27f8c1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p02-best-head-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23893329644403358, + "mae": 0.12647750973701477, + "r2": 0.6352735757827759, + "score": -0.23893329644403358 + }, + "val": { + "rmse": 0.23785555444789067, + "mae": 0.1310475766658783, + "r2": 0.6220991611480713, + "score": -0.23785555444789067 + }, + "test": { + "rmse": 0.23748897727684737, + "mae": 0.1446538269519806, + "r2": 0.6145868301391602, + "score": -0.23748897727684737 + } + }, + "time": "0:01:22.606690", + "diversity": { + "train": { + "mean_centered_corr": -0.019367322325706482, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04785257205367088 + }, + "val": { + "mean_centered_corr": -0.0196691807359457, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06109632924199104 + }, + "test": { + "mean_centered_corr": -0.01985469087958336, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08116098493337631 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 29 + ], + "head_selection_time": "0:00:00.015212" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p02-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p02-evaluation/0.toml new file mode 100644 index 000000000..93fff1a04 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p02-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.02 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p02-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p02-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p02-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p02-evaluation/0/report.json new file mode 100644 index 000000000..61f313a8f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p02-evaluation/0/report.json @@ -0,0 +1,654 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2325909984973762, + "mae": 0.12197329849004745, + "r2": 0.6543793678283691, + "score": -0.2325909984973762 + }, + "val": { + "rmse": 0.23537506682607195, + "mae": 0.12669186294078827, + "r2": 0.6299399733543396, + "score": -0.23537506682607195 + }, + "test": { + "rmse": 0.23579029486809283, + "mae": 0.14991408586502075, + "r2": 0.6200805902481079, + "score": -0.23579029486809283 + } + }, + "time": "0:01:22.606690", + "diversity": { + "train": { + "mean_centered_corr": -0.019367322325706482, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04785257205367088 + }, + "val": { + "mean_centered_corr": -0.0196691807359457, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06109632924199104 + }, + "test": { + "mean_centered_corr": -0.01985469087958336, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08116098493337631 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p02-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p02-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p02-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p02-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..eaf94028d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p02-greedy-heads-evaluation/0/report.json @@ -0,0 +1,667 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23476379254176588, + "mae": 0.12291711568832397, + "r2": 0.6478918194770813, + "score": -0.23476379254176588 + }, + "val": { + "rmse": 0.23340542834677785, + "mae": 0.12670302391052246, + "r2": 0.6361074447631836, + "score": -0.23340542834677785 + }, + "test": { + "rmse": 0.23829137280333715, + "mae": 0.15507613122463226, + "r2": 0.6119781136512756, + "score": -0.23829137280333715 + } + }, + "time": "0:01:22.606690", + "diversity": { + "train": { + "mean_centered_corr": -0.019367322325706482, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04785257205367088 + }, + "val": { + "mean_centered_corr": -0.0196691807359457, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06109632924199104 + }, + "test": { + "mean_centered_corr": -0.01985469087958336, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08116098493337631 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 29, + 25, + 28, + 0, + 16, + 21, + 4, + 25, + 1, + 6 + ], + "head_selection_time": "0:00:00.171698" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..ee0b0e618 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22949952458766654, + "mae": 0.1217280700802803, + "r2": 0.6635059118270874, + "score": -0.22949952458766654 + }, + "val": { + "rmse": 0.23798803361544055, + "mae": 0.1329043060541153, + "r2": 0.6216781139373779, + "score": -0.23798803361544055 + }, + "test": { + "rmse": 0.23971064008247925, + "mae": 0.14922766387462616, + "r2": 0.6073422431945801, + "score": -0.23971064008247925 + } + }, + "time": "0:01:27.946940", + "diversity": { + "train": { + "mean_centered_corr": -0.018655812367796898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04690698906779289 + }, + "val": { + "mean_centered_corr": -0.019464505836367607, + "mean_pairwise_disagreement": NaN, + "member_std": 0.062106743454933167 + }, + "test": { + "mean_centered_corr": -0.01887187734246254, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08776523917913437 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21 + ], + "head_selection_time": "0:00:00.015638" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p001-evaluation/0.toml new file mode 100644 index 000000000..fb28a1b73 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p001-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..643ffc8ef --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p001-evaluation/0/report.json @@ -0,0 +1,654 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22223358862007636, + "mae": 0.11760532855987549, + "r2": 0.6844753623008728, + "score": -0.22223358862007636 + }, + "val": { + "rmse": 0.23492518063329523, + "mae": 0.12833383679389954, + "r2": 0.6313532590866089, + "score": -0.23492518063329523 + }, + "test": { + "rmse": 0.23863885185928208, + "mae": 0.15527929365634918, + "r2": 0.610845685005188, + "score": -0.23863885185928208 + } + }, + "time": "0:01:27.946940", + "diversity": { + "train": { + "mean_centered_corr": -0.018655812367796898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04690698906779289 + }, + "val": { + "mean_centered_corr": -0.019464505836367607, + "mean_pairwise_disagreement": NaN, + "member_std": 0.062106743454933167 + }, + "test": { + "mean_centered_corr": -0.01887187734246254, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08776523917913437 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..98afe708a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,663 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2249769437420875, + "mae": 0.11708378046751022, + "r2": 0.6766372919082642, + "score": -0.2249769437420875 + }, + "val": { + "rmse": 0.23342205869781213, + "mae": 0.12662439048290253, + "r2": 0.636055588722229, + "score": -0.23342205869781213 + }, + "test": { + "rmse": 0.23945135735112927, + "mae": 0.1564028114080429, + "r2": 0.608191192150116, + "score": -0.23945135735112927 + } + }, + "time": "0:01:27.946940", + "diversity": { + "train": { + "mean_centered_corr": -0.018655812367796898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04690698906779289 + }, + "val": { + "mean_centered_corr": -0.019464505836367607, + "mean_pairwise_disagreement": NaN, + "member_std": 0.062106743454933167 + }, + "test": { + "mean_centered_corr": -0.01887187734246254, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08776523917913437 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21, + 4, + 8, + 25, + 22, + 7 + ], + "head_selection_time": "0:00:00.101885" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..94b39e407 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23627931705607993, + "mae": 0.12559163570404053, + "r2": 0.6433310508728027, + "score": -0.23627931705607993 + }, + "val": { + "rmse": 0.23811064537966578, + "mae": 0.13582253456115723, + "r2": 0.6212881803512573, + "score": -0.23811064537966578 + }, + "test": { + "rmse": 0.24336730552230365, + "mae": 0.16211101412773132, + "r2": 0.5952712297439575, + "score": -0.24336730552230365 + } + }, + "time": "0:01:24.069070", + "diversity": { + "train": { + "mean_centered_corr": -0.018960177898406982, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04602884501218796 + }, + "val": { + "mean_centered_corr": -0.01958228461444378, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06140345335006714 + }, + "test": { + "mean_centered_corr": -0.017305321991443634, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08274664729833603 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 22 + ], + "head_selection_time": "0:00:00.015270" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p005-evaluation/0.toml new file mode 100644 index 000000000..ba5d2c125 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p005-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..ee3e54b8c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p005-evaluation/0/report.json @@ -0,0 +1,654 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23267118722425764, + "mae": 0.12135601043701172, + "r2": 0.6541409492492676, + "score": -0.23267118722425764 + }, + "val": { + "rmse": 0.23501775344788706, + "mae": 0.12565036118030548, + "r2": 0.6310626268386841, + "score": -0.23501775344788706 + }, + "test": { + "rmse": 0.23276117546354494, + "mae": 0.1444091498851776, + "r2": 0.6297793388366699, + "score": -0.23276117546354494 + } + }, + "time": "0:01:24.069070", + "diversity": { + "train": { + "mean_centered_corr": -0.018960177898406982, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04602884501218796 + }, + "val": { + "mean_centered_corr": -0.01958228461444378, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06140345335006714 + }, + "test": { + "mean_centered_corr": -0.017305321991443634, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08274664729833603 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..d116f6601 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,668 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23364578292705543, + "mae": 0.12276748567819595, + "r2": 0.6512374877929688, + "score": -0.23364578292705543 + }, + "val": { + "rmse": 0.23317303008072252, + "mae": 0.12586387991905212, + "r2": 0.6368317604064941, + "score": -0.23317303008072252 + }, + "test": { + "rmse": 0.23484075680143124, + "mae": 0.14914271235466003, + "r2": 0.6231343746185303, + "score": -0.23484075680143124 + } + }, + "time": "0:01:24.069070", + "diversity": { + "train": { + "mean_centered_corr": -0.018960177898406982, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04602884501218796 + }, + "val": { + "mean_centered_corr": -0.01958228461444378, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06140345335006714 + }, + "test": { + "mean_centered_corr": -0.017305321991443634, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08274664729833603 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 22, + 21, + 25, + 4, + 29, + 0, + 27, + 6, + 25, + 21, + 19 + ], + "head_selection_time": "0:00:00.174870" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..33dd4e5f3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23695884432306313, + "mae": 0.12806488573551178, + "r2": 0.6412765979766846, + "score": -0.23695884432306313 + }, + "val": { + "rmse": 0.23912916812165463, + "mae": 0.1332480013370514, + "r2": 0.6180413365364075, + "score": -0.23912916812165463 + }, + "test": { + "rmse": 0.23873190398066874, + "mae": 0.14833378791809082, + "r2": 0.6105421185493469, + "score": -0.23873190398066874 + } + }, + "time": "0:01:23.755099", + "diversity": { + "train": { + "mean_centered_corr": -0.018932092934846878, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046318743377923965 + }, + "val": { + "mean_centered_corr": -0.020040452480316162, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06007201597094536 + }, + "test": { + "mean_centered_corr": -0.017600608989596367, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0855417475104332 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 29 + ], + "head_selection_time": "0:00:00.015574" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p01-evaluation/0.toml new file mode 100644 index 000000000..05d2b7359 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p01-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..2015657ce --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p01-evaluation/0/report.json @@ -0,0 +1,654 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23164681624168468, + "mae": 0.12229707092046738, + "r2": 0.6571797132492065, + "score": -0.23164681624168468 + }, + "val": { + "rmse": 0.23514059116024372, + "mae": 0.12843270599842072, + "r2": 0.6306768655776978, + "score": -0.23514059116024372 + }, + "test": { + "rmse": 0.23749547910151964, + "mae": 0.15404725074768066, + "r2": 0.6145657300949097, + "score": -0.23749547910151964 + } + }, + "time": "0:01:23.755099", + "diversity": { + "train": { + "mean_centered_corr": -0.018932092934846878, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046318743377923965 + }, + "val": { + "mean_centered_corr": -0.020040452480316162, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06007201597094536 + }, + "test": { + "mean_centered_corr": -0.017600608989596367, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0855417475104332 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..eb460972f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,667 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.232880388512877, + "mae": 0.12257546931505203, + "r2": 0.653518795967102, + "score": -0.232880388512877 + }, + "val": { + "rmse": 0.2333667286677465, + "mae": 0.1266324818134308, + "r2": 0.636228084564209, + "score": -0.2333667286677465 + }, + "test": { + "rmse": 0.2358601642328826, + "mae": 0.15086041390895844, + "r2": 0.6198554039001465, + "score": -0.2358601642328826 + } + }, + "time": "0:01:23.755099", + "diversity": { + "train": { + "mean_centered_corr": -0.018932092934846878, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046318743377923965 + }, + "val": { + "mean_centered_corr": -0.020040452480316162, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06007201597094536 + }, + "test": { + "mean_centered_corr": -0.017600608989596367, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0855417475104332 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 29, + 28, + 25, + 10, + 19, + 0, + 21, + 6, + 25, + 28 + ], + "head_selection_time": "0:00:00.163588" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p02-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p02-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p02-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p02-best-head-evaluation/0/report.json new file mode 100644 index 000000000..be1eea845 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p02-best-head-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.23145121152418155, + "mae": 0.1271854192018509, + "r2": 0.657758355140686, + "score": -0.23145121152418155 + }, + "val": { + "rmse": 0.23792787050471845, + "mae": 0.13454143702983856, + "r2": 0.6218693256378174, + "score": -0.23792787050471845 + }, + "test": { + "rmse": 0.24466013756612576, + "mae": 0.16113941371440887, + "r2": 0.5909597873687744, + "score": -0.24466013756612576 + } + }, + "time": "0:01:29.402227", + "diversity": { + "train": { + "mean_centered_corr": -0.018917854875326157, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04637383669614792 + }, + "val": { + "mean_centered_corr": -0.019510701298713684, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06465298682451248 + }, + "test": { + "mean_centered_corr": -0.01702488213777542, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09602542966604233 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21 + ], + "head_selection_time": "0:00:00.015386" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p02-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p02-evaluation/0.toml new file mode 100644 index 000000000..f13bcf99c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p02-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.02 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p02-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p02-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p02-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p02-evaluation/0/report.json new file mode 100644 index 000000000..59379db23 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p02-evaluation/0/report.json @@ -0,0 +1,654 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2243097379869937, + "mae": 0.12035604566335678, + "r2": 0.6785524487495422, + "score": -0.2243097379869937 + }, + "val": { + "rmse": 0.23506069808417399, + "mae": 0.12883473932743073, + "r2": 0.6309278011322021, + "score": -0.23506069808417399 + }, + "test": { + "rmse": 0.24070191052163564, + "mae": 0.15875394642353058, + "r2": 0.6040880084037781, + "score": -0.24070191052163564 + } + }, + "time": "0:01:29.402227", + "diversity": { + "train": { + "mean_centered_corr": -0.018917854875326157, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04637383669614792 + }, + "val": { + "mean_centered_corr": -0.019510701298713684, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06465298682451248 + }, + "test": { + "mean_centered_corr": -0.01702488213777542, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09602542966604233 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p02-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p02-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p02-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p02-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..7ef3e192f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p02-greedy-heads-evaluation/0/report.json @@ -0,0 +1,663 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22678944303849974, + "mae": 0.12170526385307312, + "r2": 0.6714060306549072, + "score": -0.22678944303849974 + }, + "val": { + "rmse": 0.23322026782389782, + "mae": 0.12833212316036224, + "r2": 0.6366845369338989, + "score": -0.23322026782389782 + }, + "test": { + "rmse": 0.23659461735657236, + "mae": 0.1518249213695526, + "r2": 0.6174842119216919, + "score": -0.23659461735657236 + } + }, + "time": "0:01:29.402227", + "diversity": { + "train": { + "mean_centered_corr": -0.018917854875326157, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04637383669614792 + }, + "val": { + "mean_centered_corr": -0.019510701298713684, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06465298682451248 + }, + "test": { + "mean_centered_corr": -0.01702488213777542, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09602542966604233 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21, + 29, + 8, + 25, + 19, + 1 + ], + "head_selection_time": "0:00:00.103468" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..41a098b0d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2282022544611775, + "mae": 0.12789320945739746, + "r2": 0.6672992706298828, + "score": -0.2282022544611775 + }, + "val": { + "rmse": 0.24061645579340615, + "mae": 0.14481423795223236, + "r2": 0.6132752895355225, + "score": -0.24061645579340615 + }, + "test": { + "rmse": 0.2630298852149786, + "mae": 0.1882508099079132, + "r2": 0.5272301435470581, + "score": -0.2630298852149786 + } + }, + "time": "0:01:29.554714", + "diversity": { + "train": { + "mean_centered_corr": -0.019097518175840378, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047171760350465775 + }, + "val": { + "mean_centered_corr": -0.01874253898859024, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0679805725812912 + }, + "test": { + "mean_centered_corr": -0.016998732462525368, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08968250453472137 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015426" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p001-evaluation/0.toml new file mode 100644 index 000000000..afea6e51a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p001-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..09db8da3b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p001-evaluation/0/report.json @@ -0,0 +1,654 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2217449394684032, + "mae": 0.11953334510326385, + "r2": 0.685861349105835, + "score": -0.2217449394684032 + }, + "val": { + "rmse": 0.23558290278789318, + "mae": 0.12969213724136353, + "r2": 0.6292861700057983, + "score": -0.23558290278789318 + }, + "test": { + "rmse": 0.24195079618816864, + "mae": 0.16031414270401, + "r2": 0.5999689102172852, + "score": -0.24195079618816864 + } + }, + "time": "0:01:29.554714", + "diversity": { + "train": { + "mean_centered_corr": -0.019097518175840378, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047171760350465775 + }, + "val": { + "mean_centered_corr": -0.01874253898859024, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0679805725812912 + }, + "test": { + "mean_centered_corr": -0.016998732462525368, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08968250453472137 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..5a30a8506 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,661 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22122007777848465, + "mae": 0.11975695192813873, + "r2": 0.6873466968536377, + "score": -0.22122007777848465 + }, + "val": { + "rmse": 0.23376432971584016, + "mae": 0.12689536809921265, + "r2": 0.634987473487854, + "score": -0.23376432971584016 + }, + "test": { + "rmse": 0.24301364183447563, + "mae": 0.1616802215576172, + "r2": 0.596446692943573, + "score": -0.24301364183447563 + } + }, + "time": "0:01:29.554714", + "diversity": { + "train": { + "mean_centered_corr": -0.019097518175840378, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047171760350465775 + }, + "val": { + "mean_centered_corr": -0.01874253898859024, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0679805725812912 + }, + "test": { + "mean_centered_corr": -0.016998732462525368, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08968250453472137 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 21, + 25, + 19 + ], + "head_selection_time": "0:00:00.074750" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..1dd4c094f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23952885281283356, + "mae": 0.12619903683662415, + "r2": 0.6334531307220459, + "score": -0.23952885281283356 + }, + "val": { + "rmse": 0.23885129531058336, + "mae": 0.13529641926288605, + "r2": 0.6189285516738892, + "score": -0.23885129531058336 + }, + "test": { + "rmse": 0.24654667666950852, + "mae": 0.16894596815109253, + "r2": 0.5846273899078369, + "score": -0.24654667666950852 + } + }, + "time": "0:01:20.972441", + "diversity": { + "train": { + "mean_centered_corr": -0.019369952380657196, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046703703701496124 + }, + "val": { + "mean_centered_corr": -0.018561262637376785, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06341557204723358 + }, + "test": { + "mean_centered_corr": -0.01718106120824814, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08355174213647842 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015400" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p005-evaluation/0.toml new file mode 100644 index 000000000..e423f0302 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p005-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..e341e65b1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p005-evaluation/0/report.json @@ -0,0 +1,654 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23157858733487574, + "mae": 0.1211637556552887, + "r2": 0.6573815941810608, + "score": -0.23157858733487574 + }, + "val": { + "rmse": 0.23531810633502007, + "mae": 0.125682532787323, + "r2": 0.6301190853118896, + "score": -0.23531810633502007 + }, + "test": { + "rmse": 0.2327283473545216, + "mae": 0.1438872069120407, + "r2": 0.6298837661743164, + "score": -0.2327283473545216 + } + }, + "time": "0:01:20.972441", + "diversity": { + "train": { + "mean_centered_corr": -0.019369952380657196, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046703703701496124 + }, + "val": { + "mean_centered_corr": -0.018561262637376785, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06341557204723358 + }, + "test": { + "mean_centered_corr": -0.01718106120824814, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08355174213647842 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..3853fa9e7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,665 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2322140894845545, + "mae": 0.12190577387809753, + "r2": 0.6554985642433167, + "score": -0.2322140894845545 + }, + "val": { + "rmse": 0.23374365968826152, + "mae": 0.12585988640785217, + "r2": 0.6350520253181458, + "score": -0.23374365968826152 + }, + "test": { + "rmse": 0.23258765102786882, + "mae": 0.14328286051750183, + "r2": 0.6303311586380005, + "score": -0.23258765102786882 + } + }, + "time": "0:01:20.972441", + "diversity": { + "train": { + "mean_centered_corr": -0.019369952380657196, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046703703701496124 + }, + "val": { + "mean_centered_corr": -0.018561262637376785, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06341557204723358 + }, + "test": { + "mean_centered_corr": -0.01718106120824814, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08355174213647842 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 21, + 0, + 25, + 29, + 6, + 19, + 28 + ], + "head_selection_time": "0:00:00.130802" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..588e93cc8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23598866476350114, + "mae": 0.13086389005184174, + "r2": 0.6442080140113831, + "score": -0.23598866476350114 + }, + "val": { + "rmse": 0.23798534123823967, + "mae": 0.13590504229068756, + "r2": 0.6216866970062256, + "score": -0.23798534123823967 + }, + "test": { + "rmse": 0.2395424764805503, + "mae": 0.1539018154144287, + "r2": 0.6078929305076599, + "score": -0.2395424764805503 + } + }, + "time": "0:01:18.608008", + "diversity": { + "train": { + "mean_centered_corr": -0.01912665367126465, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045948561280965805 + }, + "val": { + "mean_centered_corr": -0.01955518126487732, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05995342880487442 + }, + "test": { + "mean_centered_corr": -0.015723563730716705, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08670108765363693 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 16 + ], + "head_selection_time": "0:00:00.015223" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p01-evaluation/0.toml new file mode 100644 index 000000000..4c97f04e8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p01-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..060cded4a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p01-evaluation/0/report.json @@ -0,0 +1,654 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23216121546628654, + "mae": 0.12173006683588028, + "r2": 0.6556554436683655, + "score": -0.23216121546628654 + }, + "val": { + "rmse": 0.23480153213840635, + "mae": 0.12647123634815216, + "r2": 0.6317412257194519, + "score": -0.23480153213840635 + }, + "test": { + "rmse": 0.23483096909658374, + "mae": 0.14862899482250214, + "r2": 0.6231657266616821, + "score": -0.23483096909658374 + } + }, + "time": "0:01:18.608008", + "diversity": { + "train": { + "mean_centered_corr": -0.01912665367126465, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045948561280965805 + }, + "val": { + "mean_centered_corr": -0.01955518126487732, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05995342880487442 + }, + "test": { + "mean_centered_corr": -0.015723563730716705, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08670108765363693 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..71e10c5c5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,666 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23448659306502342, + "mae": 0.12265997380018234, + "r2": 0.6487228274345398, + "score": -0.23448659306502342 + }, + "val": { + "rmse": 0.23320259273091312, + "mae": 0.12646806240081787, + "r2": 0.6367396116256714, + "score": -0.23320259273091312 + }, + "test": { + "rmse": 0.23641549221562166, + "mae": 0.15262219309806824, + "r2": 0.61806321144104, + "score": -0.23641549221562166 + } + }, + "time": "0:01:18.608008", + "diversity": { + "train": { + "mean_centered_corr": -0.01912665367126465, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045948561280965805 + }, + "val": { + "mean_centered_corr": -0.01955518126487732, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05995342880487442 + }, + "test": { + "mean_centered_corr": -0.015723563730716705, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08670108765363693 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 16, + 1, + 28, + 0, + 29, + 21, + 25, + 1, + 4 + ], + "head_selection_time": "0:00:00.145758" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p02-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p02-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p02-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p02-best-head-evaluation/0/report.json new file mode 100644 index 000000000..be1e3d331 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p02-best-head-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.23020776422064904, + "mae": 0.1262739598751068, + "r2": 0.6614258289337158, + "score": -0.23020776422064904 + }, + "val": { + "rmse": 0.23771204736707321, + "mae": 0.13801762461662292, + "r2": 0.6225550174713135, + "score": -0.23771204736707321 + }, + "test": { + "rmse": 0.24308994802985293, + "mae": 0.16311529278755188, + "r2": 0.5961931943893433, + "score": -0.24308994802985293 + } + }, + "time": "0:01:27.894088", + "diversity": { + "train": { + "mean_centered_corr": -0.018592098727822304, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045505620539188385 + }, + "val": { + "mean_centered_corr": -0.018926210701465607, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06432383507490158 + }, + "test": { + "mean_centered_corr": -0.012847395613789558, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09277871996164322 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 22 + ], + "head_selection_time": "0:00:00.015561" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p02-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p02-evaluation/0.toml new file mode 100644 index 000000000..ff31b4334 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p02-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.02 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p02-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p02-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p02-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p02-evaluation/0/report.json new file mode 100644 index 000000000..fa6fecdb7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p02-evaluation/0/report.json @@ -0,0 +1,654 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22398877861023744, + "mae": 0.1194881871342659, + "r2": 0.6794716715812683, + "score": -0.22398877861023744 + }, + "val": { + "rmse": 0.23510019641837185, + "mae": 0.12923409044742584, + "r2": 0.6308038234710693, + "score": -0.23510019641837185 + }, + "test": { + "rmse": 0.23935828921252758, + "mae": 0.15663591027259827, + "r2": 0.6084957122802734, + "score": -0.23935828921252758 + } + }, + "time": "0:01:27.894088", + "diversity": { + "train": { + "mean_centered_corr": -0.018592098727822304, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045505620539188385 + }, + "val": { + "mean_centered_corr": -0.018926210701465607, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06432383507490158 + }, + "test": { + "mean_centered_corr": -0.012847395613789558, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09277871996164322 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p02-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p02-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p02-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p02-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..47d385ff1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p02-greedy-heads-evaluation/0/report.json @@ -0,0 +1,663 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22544924405132602, + "mae": 0.12180135399103165, + "r2": 0.6752781867980957, + "score": -0.22544924405132602 + }, + "val": { + "rmse": 0.2334750859643239, + "mae": 0.12858723104000092, + "r2": 0.6358902454376221, + "score": -0.2334750859643239 + }, + "test": { + "rmse": 0.23518646749461922, + "mae": 0.15039512515068054, + "r2": 0.6220239400863647, + "score": -0.23518646749461922 + } + }, + "time": "0:01:27.894088", + "diversity": { + "train": { + "mean_centered_corr": -0.018592098727822304, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045505620539188385 + }, + "val": { + "mean_centered_corr": -0.018926210701465607, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06432383507490158 + }, + "test": { + "mean_centered_corr": -0.012847395613789558, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09277871996164322 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 22, + 6, + 19, + 25, + 21, + 29 + ], + "head_selection_time": "0:00:00.103487" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..138288004 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24049199255343762, + "mae": 0.12693814933300018, + "r2": 0.6304994225502014, + "score": -0.24049199255343762 + }, + "val": { + "rmse": 0.23716996532706527, + "mae": 0.13215987384319305, + "r2": 0.624274492263794, + "score": -0.23716996532706527 + }, + "test": { + "rmse": 0.23470593771148443, + "mae": 0.14771248400211334, + "r2": 0.6235669851303101, + "score": -0.23470593771148443 + } + }, + "time": "0:01:24.453355", + "diversity": { + "train": { + "mean_centered_corr": -0.0185525044798851, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045574937015771866 + }, + "val": { + "mean_centered_corr": -0.019020110368728638, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05756770819425583 + }, + "test": { + "mean_centered_corr": -0.017030077055096626, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08178099244832993 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015188" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p001-evaluation/0.toml new file mode 100644 index 000000000..daef5efee --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p001-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..c6ec0be46 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p001-evaluation/0/report.json @@ -0,0 +1,654 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23121498010281438, + "mae": 0.12121064215898514, + "r2": 0.6584566831588745, + "score": -0.23121498010281438 + }, + "val": { + "rmse": 0.23523322965051088, + "mae": 0.12707281112670898, + "r2": 0.6303858757019043, + "score": -0.23523322965051088 + }, + "test": { + "rmse": 0.23438947156097584, + "mae": 0.14751991629600525, + "r2": 0.6245813965797424, + "score": -0.23438947156097584 + } + }, + "time": "0:01:24.453355", + "diversity": { + "train": { + "mean_centered_corr": -0.0185525044798851, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045574937015771866 + }, + "val": { + "mean_centered_corr": -0.019020110368728638, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05756770819425583 + }, + "test": { + "mean_centered_corr": -0.017030077055096626, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08178099244832993 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..daaa60081 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,662 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2346798186181653, + "mae": 0.12336737662553787, + "r2": 0.6481436491012573, + "score": -0.2346798186181653 + }, + "val": { + "rmse": 0.23379921122284117, + "mae": 0.12755374610424042, + "r2": 0.6348785161972046, + "score": -0.23379921122284117 + }, + "test": { + "rmse": 0.23898482806189927, + "mae": 0.15705350041389465, + "r2": 0.6097164154052734, + "score": -0.23898482806189927 + } + }, + "time": "0:01:24.453355", + "diversity": { + "train": { + "mean_centered_corr": -0.0185525044798851, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045574937015771866 + }, + "val": { + "mean_centered_corr": -0.019020110368728638, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05756770819425583 + }, + "test": { + "mean_centered_corr": -0.017030077055096626, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08178099244832993 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 25, + 21, + 0, + 1 + ], + "head_selection_time": "0:00:00.088168" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..b8bf7c2d1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2368178141443934, + "mae": 0.12198712676763535, + "r2": 0.6417034864425659, + "score": -0.2368178141443934 + }, + "val": { + "rmse": 0.23907342929415865, + "mae": 0.13113287091255188, + "r2": 0.6182193756103516, + "score": -0.23907342929415865 + }, + "test": { + "rmse": 0.23310698183908637, + "mae": 0.14301016926765442, + "r2": 0.6286784410476685, + "score": -0.23310698183908637 + } + }, + "time": "0:01:21.540934", + "diversity": { + "train": { + "mean_centered_corr": -0.018635282292962074, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04599732533097267 + }, + "val": { + "mean_centered_corr": -0.01881239004433155, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06206817552447319 + }, + "test": { + "mean_centered_corr": -0.017686480656266212, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08751651644706726 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 28 + ], + "head_selection_time": "0:00:00.015354" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p005-evaluation/0.toml new file mode 100644 index 000000000..c35d423aa --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p005-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..8bf55312b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p005-evaluation/0/report.json @@ -0,0 +1,654 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23194337523062297, + "mae": 0.12280548363924026, + "r2": 0.6563013792037964, + "score": -0.23194337523062297 + }, + "val": { + "rmse": 0.23519216180885402, + "mae": 0.1276979148387909, + "r2": 0.6305148601531982, + "score": -0.23519216180885402 + }, + "test": { + "rmse": 0.23499174827687583, + "mae": 0.1487179696559906, + "r2": 0.622649610042572, + "score": -0.23499174827687583 + } + }, + "time": "0:01:21.540934", + "diversity": { + "train": { + "mean_centered_corr": -0.018635282292962074, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04599732533097267 + }, + "val": { + "mean_centered_corr": -0.01881239004433155, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06206817552447319 + }, + "test": { + "mean_centered_corr": -0.017686480656266212, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08751651644706726 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..877fe50a7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,667 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23278222079157185, + "mae": 0.12435509264469147, + "r2": 0.6538107991218567, + "score": -0.23278222079157185 + }, + "val": { + "rmse": 0.2332399859921054, + "mae": 0.12662732601165771, + "r2": 0.6366231441497803, + "score": -0.2332399859921054 + }, + "test": { + "rmse": 0.2326072306387775, + "mae": 0.14432717859745026, + "r2": 0.6302688717842102, + "score": -0.2326072306387775 + } + }, + "time": "0:01:21.540934", + "diversity": { + "train": { + "mean_centered_corr": -0.018635282292962074, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04599732533097267 + }, + "val": { + "mean_centered_corr": -0.01881239004433155, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06206817552447319 + }, + "test": { + "mean_centered_corr": -0.017686480656266212, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08751651644706726 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 28, + 25, + 29, + 21, + 0, + 19, + 20, + 25, + 1, + 6 + ], + "head_selection_time": "0:00:00.170893" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..bfbb3aa87 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23711698667572978, + "mae": 0.1269228607416153, + "r2": 0.6407976150512695, + "score": -0.23711698667572978 + }, + "val": { + "rmse": 0.2376162677316268, + "mae": 0.13255804777145386, + "r2": 0.6228591203689575, + "score": -0.2376162677316268 + }, + "test": { + "rmse": 0.23517437354931303, + "mae": 0.14232948422431946, + "r2": 0.6220628023147583, + "score": -0.23517437354931303 + } + }, + "time": "0:01:21.799996", + "diversity": { + "train": { + "mean_centered_corr": -0.01871204562485218, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04567575082182884 + }, + "val": { + "mean_centered_corr": -0.01849065162241459, + "mean_pairwise_disagreement": NaN, + "member_std": 0.058917272835969925 + }, + "test": { + "mean_centered_corr": -0.016842441633343697, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08555692434310913 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 29 + ], + "head_selection_time": "0:00:00.017414" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p01-evaluation/0.toml new file mode 100644 index 000000000..60de6d162 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p01-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..71b8eb184 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p01-evaluation/0/report.json @@ -0,0 +1,654 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23218298905099302, + "mae": 0.12276018410921097, + "r2": 0.6555908918380737, + "score": -0.23218298905099302 + }, + "val": { + "rmse": 0.23534793771825302, + "mae": 0.12849925458431244, + "r2": 0.6300252676010132, + "score": -0.23534793771825302 + }, + "test": { + "rmse": 0.2358765820307155, + "mae": 0.1505100429058075, + "r2": 0.6198024749755859, + "score": -0.2358765820307155 + } + }, + "time": "0:01:21.799996", + "diversity": { + "train": { + "mean_centered_corr": -0.01871204562485218, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04567575082182884 + }, + "val": { + "mean_centered_corr": -0.01849065162241459, + "mean_pairwise_disagreement": NaN, + "member_std": 0.058917272835969925 + }, + "test": { + "mean_centered_corr": -0.016842441633343697, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08555692434310913 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..ea4c5da7c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,665 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23342471593298392, + "mae": 0.1249859556555748, + "r2": 0.6518971920013428, + "score": -0.23342471593298392 + }, + "val": { + "rmse": 0.2335073783441898, + "mae": 0.12728865444660187, + "r2": 0.6357895135879517, + "score": -0.2335073783441898 + }, + "test": { + "rmse": 0.2329070852784964, + "mae": 0.1443146914243698, + "r2": 0.6293150186538696, + "score": -0.2329070852784964 + } + }, + "time": "0:01:21.799996", + "diversity": { + "train": { + "mean_centered_corr": -0.01871204562485218, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04567575082182884 + }, + "val": { + "mean_centered_corr": -0.01849065162241459, + "mean_pairwise_disagreement": NaN, + "member_std": 0.058917272835969925 + }, + "test": { + "mean_centered_corr": -0.016842441633343697, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08555692434310913 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 29, + 0, + 28, + 19, + 25, + 13, + 21, + 0 + ], + "head_selection_time": "0:00:00.138920" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p02-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p02-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p02-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p02-best-head-evaluation/0/report.json new file mode 100644 index 000000000..8b6bd1cf1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p02-best-head-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.234490000803403, + "mae": 0.12728147208690643, + "r2": 0.6487126350402832, + "score": -0.234490000803403 + }, + "val": { + "rmse": 0.23720560242248867, + "mae": 0.1349465399980545, + "r2": 0.6241616010665894, + "score": -0.23720560242248867 + }, + "test": { + "rmse": 0.23787240614757268, + "mae": 0.1500697135925293, + "r2": 0.6133413314819336, + "score": -0.23787240614757268 + } + }, + "time": "0:01:25.321284", + "diversity": { + "train": { + "mean_centered_corr": -0.018023740500211716, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047287896275520325 + }, + "val": { + "mean_centered_corr": -0.01780591532588005, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06861133128404617 + }, + "test": { + "mean_centered_corr": -0.015070421621203423, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09827303886413574 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 29 + ], + "head_selection_time": "0:00:00.015329" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p02-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p02-evaluation/0.toml new file mode 100644 index 000000000..5bb2a3fa1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p02-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.02 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p02-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p02-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p02-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p02-evaluation/0/report.json new file mode 100644 index 000000000..9a822931b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p02-evaluation/0/report.json @@ -0,0 +1,654 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2240611226745793, + "mae": 0.11743048578500748, + "r2": 0.6792646050453186, + "score": -0.2240611226745793 + }, + "val": { + "rmse": 0.23499558463666298, + "mae": 0.12714111804962158, + "r2": 0.6311322450637817, + "score": -0.23499558463666298 + }, + "test": { + "rmse": 0.23934769790965568, + "mae": 0.15463049709796906, + "r2": 0.6085303425788879, + "score": -0.23934769790965568 + } + }, + "time": "0:01:25.321284", + "diversity": { + "train": { + "mean_centered_corr": -0.018023740500211716, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047287896275520325 + }, + "val": { + "mean_centered_corr": -0.01780591532588005, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06861133128404617 + }, + "test": { + "mean_centered_corr": -0.015070421621203423, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09827303886413574 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p02-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p02-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p02-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p02-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..d0b6cd254 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p02-greedy-heads-evaluation/0/report.json @@ -0,0 +1,665 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22719550493821902, + "mae": 0.11786691844463348, + "r2": 0.6702283024787903, + "score": -0.22719550493821902 + }, + "val": { + "rmse": 0.23350963577191214, + "mae": 0.1265878677368164, + "r2": 0.6357824802398682, + "score": -0.23350963577191214 + }, + "test": { + "rmse": 0.2393402580368093, + "mae": 0.15526840090751648, + "r2": 0.6085547208786011, + "score": -0.2393402580368093 + } + }, + "time": "0:01:25.321284", + "diversity": { + "train": { + "mean_centered_corr": -0.018023740500211716, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047287896275520325 + }, + "val": { + "mean_centered_corr": -0.01780591532588005, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06861133128404617 + }, + "test": { + "mean_centered_corr": -0.015070421621203423, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09827303886413574 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 29, + 10, + 7, + 22, + 4, + 8, + 25, + 21 + ], + "head_selection_time": "0:00:00.130729" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..bafb9519d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22820973096491176, + "mae": 0.12737685441970825, + "r2": 0.667277455329895, + "score": -0.22820973096491176 + }, + "val": { + "rmse": 0.23905700508983188, + "mae": 0.14095449447631836, + "r2": 0.6182718276977539, + "score": -0.23905700508983188 + }, + "test": { + "rmse": 0.2554892709638668, + "mae": 0.17996472120285034, + "r2": 0.5539485812187195, + "score": -0.2554892709638668 + } + }, + "time": "0:02:00.734464", + "diversity": { + "train": { + "mean_centered_corr": -0.019317513331770897, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04794894531369209 + }, + "val": { + "mean_centered_corr": -0.019240468740463257, + "mean_pairwise_disagreement": NaN, + "member_std": 0.065296970307827 + }, + "test": { + "mean_centered_corr": -0.018747208639979362, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08741389214992523 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.016010" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p001-evaluation/0.toml new file mode 100644 index 000000000..29d629616 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p001-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..eba302088 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p001-evaluation/0/report.json @@ -0,0 +1,654 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22218533123207995, + "mae": 0.11801747977733612, + "r2": 0.6846123933792114, + "score": -0.22218533123207995 + }, + "val": { + "rmse": 0.23516964509902982, + "mae": 0.1285666972398758, + "r2": 0.6305856704711914, + "score": -0.23516964509902982 + }, + "test": { + "rmse": 0.23928895864366434, + "mae": 0.15676262974739075, + "r2": 0.608722448348999, + "score": -0.23928895864366434 + } + }, + "time": "0:02:00.734464", + "diversity": { + "train": { + "mean_centered_corr": -0.019317513331770897, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04794894531369209 + }, + "val": { + "mean_centered_corr": -0.019240468740463257, + "mean_pairwise_disagreement": NaN, + "member_std": 0.065296970307827 + }, + "test": { + "mean_centered_corr": -0.018747208639979362, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08741389214992523 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..ce12b44c5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,664 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22489126180626096, + "mae": 0.11878597736358643, + "r2": 0.6768835783004761, + "score": -0.22489126180626096 + }, + "val": { + "rmse": 0.23363252496566134, + "mae": 0.12608498334884644, + "r2": 0.6353989839553833, + "score": -0.23363252496566134 + }, + "test": { + "rmse": 0.2362188764537236, + "mae": 0.15149326622486115, + "r2": 0.618698239326477, + "score": -0.2362188764537236 + } + }, + "time": "0:02:00.734464", + "diversity": { + "train": { + "mean_centered_corr": -0.019317513331770897, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04794894531369209 + }, + "val": { + "mean_centered_corr": -0.019240468740463257, + "mean_pairwise_disagreement": NaN, + "member_std": 0.065296970307827 + }, + "test": { + "mean_centered_corr": -0.018747208639979362, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08741389214992523 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 8, + 22, + 7, + 29, + 25, + 21 + ], + "head_selection_time": "0:00:00.119887" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..df545e284 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.23007882803014523, + "mae": 0.12374971061944962, + "r2": 0.6618049740791321, + "score": -0.23007882803014523 + }, + "val": { + "rmse": 0.23901989843968774, + "mae": 0.13069558143615723, + "r2": 0.6183903217315674, + "score": -0.23901989843968774 + }, + "test": { + "rmse": 0.24063495640158664, + "mae": 0.1513846516609192, + "r2": 0.6043082475662231, + "score": -0.24063495640158664 + } + }, + "time": "0:02:00.389497", + "diversity": { + "train": { + "mean_centered_corr": -0.018541699275374413, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04868823289871216 + }, + "val": { + "mean_centered_corr": -0.019729774445295334, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06561318784952164 + }, + "test": { + "mean_centered_corr": -0.01752825267612934, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09573257714509964 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21 + ], + "head_selection_time": "0:00:00.015353" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p005-evaluation/0.toml new file mode 100644 index 000000000..713099f48 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p005-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..2a8f194cc --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p005-evaluation/0/report.json @@ -0,0 +1,654 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22270596936228843, + "mae": 0.11821265518665314, + "r2": 0.6831325888633728, + "score": -0.22270596936228843 + }, + "val": { + "rmse": 0.23541076200529618, + "mae": 0.1285439282655716, + "r2": 0.6298277378082275, + "score": -0.23541076200529618 + }, + "test": { + "rmse": 0.2395584319829453, + "mae": 0.15599201619625092, + "r2": 0.6078406572341919, + "score": -0.2395584319829453 + } + }, + "time": "0:02:00.389497", + "diversity": { + "train": { + "mean_centered_corr": -0.018541699275374413, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04868823289871216 + }, + "val": { + "mean_centered_corr": -0.019729774445295334, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06561318784952164 + }, + "test": { + "mean_centered_corr": -0.01752825267612934, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09573257714509964 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..bd9e463d1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,663 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22560705771393183, + "mae": 0.11890538036823273, + "r2": 0.6748234033584595, + "score": -0.22560705771393183 + }, + "val": { + "rmse": 0.23373660723675202, + "mae": 0.1272813230752945, + "r2": 0.6350740790367126, + "score": -0.23373660723675202 + }, + "test": { + "rmse": 0.23443243594536417, + "mae": 0.14499256014823914, + "r2": 0.624443769454956, + "score": -0.23443243594536417 + } + }, + "time": "0:02:00.389497", + "diversity": { + "train": { + "mean_centered_corr": -0.018541699275374413, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04868823289871216 + }, + "val": { + "mean_centered_corr": -0.019729774445295334, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06561318784952164 + }, + "test": { + "mean_centered_corr": -0.01752825267612934, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09573257714509964 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21, + 29, + 8, + 0, + 19, + 22 + ], + "head_selection_time": "0:00:00.101274" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..3f5971d21 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.23015652536762907, + "mae": 0.12545116245746613, + "r2": 0.661576509475708, + "score": -0.23015652536762907 + }, + "val": { + "rmse": 0.239085684381371, + "mae": 0.13462616503238678, + "r2": 0.6181802749633789, + "score": -0.239085684381371 + }, + "test": { + "rmse": 0.24513387433279307, + "mae": 0.16250139474868774, + "r2": 0.5893741846084595, + "score": -0.24513387433279307 + } + }, + "time": "0:01:58.386465", + "diversity": { + "train": { + "mean_centered_corr": -0.01839604787528515, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0476016141474247 + }, + "val": { + "mean_centered_corr": -0.0184602253139019, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06696488708257675 + }, + "test": { + "mean_centered_corr": -0.018153522163629532, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09845950454473495 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21 + ], + "head_selection_time": "0:00:00.014850" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p01-evaluation/0.toml new file mode 100644 index 000000000..2e5997daa --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p01-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..f9eec6cd8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p01-evaluation/0/report.json @@ -0,0 +1,654 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22416379102176556, + "mae": 0.11926887929439545, + "r2": 0.6789705753326416, + "score": -0.22416379102176556 + }, + "val": { + "rmse": 0.23516104335271928, + "mae": 0.12952451407909393, + "r2": 0.6306126713752747, + "score": -0.23516104335271928 + }, + "test": { + "rmse": 0.24165658285763553, + "mae": 0.1609659045934677, + "r2": 0.6009412407875061, + "score": -0.24165658285763553 + } + }, + "time": "0:01:58.386465", + "diversity": { + "train": { + "mean_centered_corr": -0.01839604787528515, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0476016141474247 + }, + "val": { + "mean_centered_corr": -0.0184602253139019, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06696488708257675 + }, + "test": { + "mean_centered_corr": -0.018153522163629532, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09845950454473495 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..46cbec9f2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,667 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22445475190340167, + "mae": 0.11813689768314362, + "r2": 0.6781366467475891, + "score": -0.22445475190340167 + }, + "val": { + "rmse": 0.23356035433981295, + "mae": 0.1266433298587799, + "r2": 0.6356241703033447, + "score": -0.23356035433981295 + }, + "test": { + "rmse": 0.2372102431750833, + "mae": 0.15235204994678497, + "r2": 0.6154910326004028, + "score": -0.2372102431750833 + } + }, + "time": "0:01:58.386465", + "diversity": { + "train": { + "mean_centered_corr": -0.01839604787528515, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0476016141474247 + }, + "val": { + "mean_centered_corr": -0.0184602253139019, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06696488708257675 + }, + "test": { + "mean_centered_corr": -0.018153522163629532, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09845950454473495 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21, + 7, + 4, + 22, + 19, + 8, + 18, + 7, + 29, + 25 + ], + "head_selection_time": "0:00:00.157297" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p02-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p02-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p02-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p02-best-head-evaluation/0/report.json new file mode 100644 index 000000000..c50fb3660 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p02-best-head-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23893329644403358, + "mae": 0.12647750973701477, + "r2": 0.6352735757827759, + "score": -0.23893329644403358 + }, + "val": { + "rmse": 0.23785555444789067, + "mae": 0.1310475766658783, + "r2": 0.6220991611480713, + "score": -0.23785555444789067 + }, + "test": { + "rmse": 0.23748897727684737, + "mae": 0.1446538269519806, + "r2": 0.6145868301391602, + "score": -0.23748897727684737 + } + }, + "time": "0:01:52.701352", + "diversity": { + "train": { + "mean_centered_corr": -0.019367322325706482, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04785257205367088 + }, + "val": { + "mean_centered_corr": -0.0196691807359457, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06109632924199104 + }, + "test": { + "mean_centered_corr": -0.01985469087958336, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08116098493337631 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 29 + ], + "head_selection_time": "0:00:00.016291" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p02-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p02-evaluation/0.toml new file mode 100644 index 000000000..2cb767b7b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p02-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.02 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p02-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p02-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p02-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p02-evaluation/0/report.json new file mode 100644 index 000000000..4a4d01205 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p02-evaluation/0/report.json @@ -0,0 +1,654 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2325909984973762, + "mae": 0.12197329849004745, + "r2": 0.6543793678283691, + "score": -0.2325909984973762 + }, + "val": { + "rmse": 0.23537506682607195, + "mae": 0.12669186294078827, + "r2": 0.6299399733543396, + "score": -0.23537506682607195 + }, + "test": { + "rmse": 0.23579029486809283, + "mae": 0.14991408586502075, + "r2": 0.6200805902481079, + "score": -0.23579029486809283 + } + }, + "time": "0:01:52.701352", + "diversity": { + "train": { + "mean_centered_corr": -0.019367322325706482, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04785257205367088 + }, + "val": { + "mean_centered_corr": -0.0196691807359457, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06109632924199104 + }, + "test": { + "mean_centered_corr": -0.01985469087958336, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08116098493337631 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p02-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p02-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p02-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p02-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..65532463f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p02-greedy-heads-evaluation/0/report.json @@ -0,0 +1,667 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23476379254176588, + "mae": 0.12291711568832397, + "r2": 0.6478918194770813, + "score": -0.23476379254176588 + }, + "val": { + "rmse": 0.23340542834677785, + "mae": 0.12670302391052246, + "r2": 0.6361074447631836, + "score": -0.23340542834677785 + }, + "test": { + "rmse": 0.23829137280333715, + "mae": 0.15507613122463226, + "r2": 0.6119781136512756, + "score": -0.23829137280333715 + } + }, + "time": "0:01:52.701352", + "diversity": { + "train": { + "mean_centered_corr": -0.019367322325706482, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04785257205367088 + }, + "val": { + "mean_centered_corr": -0.0196691807359457, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06109632924199104 + }, + "test": { + "mean_centered_corr": -0.01985469087958336, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08116098493337631 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 29, + 25, + 28, + 0, + 16, + 21, + 4, + 25, + 1, + 6 + ], + "head_selection_time": "0:00:00.166251" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..530b524b7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22528267991110515, + "mae": 0.12274349480867386, + "r2": 0.6757578253746033, + "score": -0.22528267991110515 + }, + "val": { + "rmse": 0.23779726122481318, + "mae": 0.13489505648612976, + "r2": 0.6222844123840332, + "score": -0.23779726122481318 + }, + "test": { + "rmse": 0.24153146027417532, + "mae": 0.15904922783374786, + "r2": 0.6013543605804443, + "score": -0.24153146027417532 + } + }, + "time": "0:01:59.320916", + "diversity": { + "train": { + "mean_centered_corr": -0.018156183883547783, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047758832573890686 + }, + "val": { + "mean_centered_corr": -0.01822330430150032, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06243854761123657 + }, + "test": { + "mean_centered_corr": -0.015858400613069534, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08906728774309158 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 22 + ], + "head_selection_time": "0:00:00.015216" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p001-evaluation/0.toml new file mode 100644 index 000000000..bc810cded --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p001-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..ce31664a1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p001-evaluation/0/report.json @@ -0,0 +1,654 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22031523790754975, + "mae": 0.11761360615491867, + "r2": 0.6898991465568542, + "score": -0.22031523790754975 + }, + "val": { + "rmse": 0.2350763793606662, + "mae": 0.12925966084003448, + "r2": 0.6308785676956177, + "score": -0.2350763793606662 + }, + "test": { + "rmse": 0.23832022238784234, + "mae": 0.15449607372283936, + "r2": 0.6118841171264648, + "score": -0.23832022238784234 + } + }, + "time": "0:01:59.320916", + "diversity": { + "train": { + "mean_centered_corr": -0.018156183883547783, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047758832573890686 + }, + "val": { + "mean_centered_corr": -0.01822330430150032, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06243854761123657 + }, + "test": { + "mean_centered_corr": -0.015858400613069534, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08906728774309158 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..74a0303ab --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,664 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2219229132672244, + "mae": 0.11750265210866928, + "r2": 0.6853569746017456, + "score": -0.2219229132672244 + }, + "val": { + "rmse": 0.23334639858951206, + "mae": 0.12622985243797302, + "r2": 0.63629150390625, + "score": -0.23334639858951206 + }, + "test": { + "rmse": 0.23179329091828219, + "mae": 0.1403104066848755, + "r2": 0.6328518390655518, + "score": -0.23179329091828219 + } + }, + "time": "0:01:59.320916", + "diversity": { + "train": { + "mean_centered_corr": -0.018156183883547783, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047758832573890686 + }, + "val": { + "mean_centered_corr": -0.01822330430150032, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06243854761123657 + }, + "test": { + "mean_centered_corr": -0.015858400613069534, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08906728774309158 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 22, + 7, + 29, + 25, + 19, + 8, + 31 + ], + "head_selection_time": "0:00:00.115448" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..92cc514d0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2386679091803034, + "mae": 0.13499177992343903, + "r2": 0.6360833644866943, + "score": -0.2386679091803034 + }, + "val": { + "rmse": 0.23814119850842008, + "mae": 0.13966409862041473, + "r2": 0.6211909651756287, + "score": -0.23814119850842008 + }, + "test": { + "rmse": 0.2663355327951239, + "mae": 0.1914888322353363, + "r2": 0.5152723789215088, + "score": -0.2663355327951239 + } + }, + "time": "0:01:52.189349", + "diversity": { + "train": { + "mean_centered_corr": -0.019499018788337708, + "mean_pairwise_disagreement": NaN, + "member_std": 0.044644709676504135 + }, + "val": { + "mean_centered_corr": -0.019907629117369652, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05747465789318085 + }, + "test": { + "mean_centered_corr": -0.016552036628127098, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08085940778255463 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25 + ], + "head_selection_time": "0:00:00.015272" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p005-evaluation/0.toml new file mode 100644 index 000000000..c530ba33a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p005-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..1d7b35844 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p005-evaluation/0/report.json @@ -0,0 +1,654 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23138006715156628, + "mae": 0.12110890448093414, + "r2": 0.6579687595367432, + "score": -0.23138006715156628 + }, + "val": { + "rmse": 0.2350520527289723, + "mae": 0.12721683084964752, + "r2": 0.6309549808502197, + "score": -0.2350520527289723 + }, + "test": { + "rmse": 0.23495368248701565, + "mae": 0.14864283800125122, + "r2": 0.6227718591690063, + "score": -0.23495368248701565 + } + }, + "time": "0:01:52.189349", + "diversity": { + "train": { + "mean_centered_corr": -0.019499018788337708, + "mean_pairwise_disagreement": NaN, + "member_std": 0.044644709676504135 + }, + "val": { + "mean_centered_corr": -0.019907629117369652, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05747465789318085 + }, + "test": { + "mean_centered_corr": -0.016552036628127098, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08085940778255463 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..1896aec73 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,663 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2324669097097326, + "mae": 0.12446025013923645, + "r2": 0.6547480225563049, + "score": -0.2324669097097326 + }, + "val": { + "rmse": 0.23343416363529057, + "mae": 0.12728102505207062, + "r2": 0.6360177993774414, + "score": -0.23343416363529057 + }, + "test": { + "rmse": 0.234433762813203, + "mae": 0.14741288125514984, + "r2": 0.6244394779205322, + "score": -0.234433762813203 + } + }, + "time": "0:01:52.189349", + "diversity": { + "train": { + "mean_centered_corr": -0.019499018788337708, + "mean_pairwise_disagreement": NaN, + "member_std": 0.044644709676504135 + }, + "val": { + "mean_centered_corr": -0.019907629117369652, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05747465789318085 + }, + "test": { + "mean_centered_corr": -0.016552036628127098, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08085940778255463 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25, + 21, + 0, + 4, + 6, + 19 + ], + "head_selection_time": "0:00:00.103853" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..3e1dabc20 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23445258445893527, + "mae": 0.12787146866321564, + "r2": 0.6488246917724609, + "score": -0.23445258445893527 + }, + "val": { + "rmse": 0.23744052562658574, + "mae": 0.1366073340177536, + "r2": 0.6234167814254761, + "score": -0.23744052562658574 + }, + "test": { + "rmse": 0.2382342886377602, + "mae": 0.15353862941265106, + "r2": 0.6121640205383301, + "score": -0.2382342886377602 + } + }, + "time": "0:01:52.520601", + "diversity": { + "train": { + "mean_centered_corr": -0.019367916509509087, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04525168240070343 + }, + "val": { + "mean_centered_corr": -0.019405901432037354, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05762277543544769 + }, + "test": { + "mean_centered_corr": -0.018559793010354042, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07860848307609558 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 16 + ], + "head_selection_time": "0:00:00.015170" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p01-evaluation/0.toml new file mode 100644 index 000000000..4549a6ec0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p01-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..d9d852c8d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p01-evaluation/0/report.json @@ -0,0 +1,654 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23102999093285262, + "mae": 0.1207849308848381, + "r2": 0.659002959728241, + "score": -0.23102999093285262 + }, + "val": { + "rmse": 0.23466412671675002, + "mae": 0.12721090018749237, + "r2": 0.6321721076965332, + "score": -0.23466412671675002 + }, + "test": { + "rmse": 0.2324620941369229, + "mae": 0.14480625092983246, + "r2": 0.630730152130127, + "score": -0.2324620941369229 + } + }, + "time": "0:01:52.520601", + "diversity": { + "train": { + "mean_centered_corr": -0.019367916509509087, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04525168240070343 + }, + "val": { + "mean_centered_corr": -0.019405901432037354, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05762277543544769 + }, + "test": { + "mean_centered_corr": -0.018559793010354042, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07860848307609558 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..1ea4d8c63 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,673 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23228452935964564, + "mae": 0.12128094583749771, + "r2": 0.6552895307540894, + "score": -0.23228452935964564 + }, + "val": { + "rmse": 0.23321172995482362, + "mae": 0.12664632499217987, + "r2": 0.6367111802101135, + "score": -0.23321172995482362 + }, + "test": { + "rmse": 0.234184776040884, + "mae": 0.1486988514661789, + "r2": 0.6252368688583374, + "score": -0.234184776040884 + } + }, + "time": "0:01:52.520601", + "diversity": { + "train": { + "mean_centered_corr": -0.019367916509509087, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04525168240070343 + }, + "val": { + "mean_centered_corr": -0.019405901432037354, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05762277543544769 + }, + "test": { + "mean_centered_corr": -0.018559793010354042, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07860848307609558 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 16, + 21, + 25, + 29, + 10, + 4, + 25, + 15, + 0, + 19, + 21, + 25, + 29, + 28, + 25, + 21 + ], + "head_selection_time": "0:00:00.249510" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p02-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p02-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p02-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p02-best-head-evaluation/0/report.json new file mode 100644 index 000000000..bf2abe48d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p02-best-head-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23954623995992808, + "mae": 0.12636718153953552, + "r2": 0.6333998441696167, + "score": -0.23954623995992808 + }, + "val": { + "rmse": 0.23869658059774151, + "mae": 0.1339649260044098, + "r2": 0.6194220185279846, + "score": -0.23869658059774151 + }, + "test": { + "rmse": 0.24052100411353006, + "mae": 0.1552955061197281, + "r2": 0.6046829223632812, + "score": -0.24052100411353006 + } + }, + "time": "0:01:52.324536", + "diversity": { + "train": { + "mean_centered_corr": -0.018679402768611908, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04577600955963135 + }, + "val": { + "mean_centered_corr": -0.019475068897008896, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05786792188882828 + }, + "test": { + "mean_centered_corr": -0.017192553728818893, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07846961915493011 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015515" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p02-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p02-evaluation/0.toml new file mode 100644 index 000000000..c4040812f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p02-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.02 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p02-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p02-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p02-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p02-evaluation/0/report.json new file mode 100644 index 000000000..29506ab1d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p02-evaluation/0/report.json @@ -0,0 +1,654 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2307699869541623, + "mae": 0.12244833260774612, + "r2": 0.6597700119018555, + "score": -0.2307699869541623 + }, + "val": { + "rmse": 0.23532386870106714, + "mae": 0.12836803495883942, + "r2": 0.6301009654998779, + "score": -0.23532386870106714 + }, + "test": { + "rmse": 0.23728542485175996, + "mae": 0.1531296819448471, + "r2": 0.6152472496032715, + "score": -0.23728542485175996 + } + }, + "time": "0:01:52.324536", + "diversity": { + "train": { + "mean_centered_corr": -0.018679402768611908, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04577600955963135 + }, + "val": { + "mean_centered_corr": -0.019475068897008896, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05786792188882828 + }, + "test": { + "mean_centered_corr": -0.017192553728818893, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07846961915493011 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p02-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p02-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p02-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p02-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..adff072f4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p02-greedy-heads-evaluation/0/report.json @@ -0,0 +1,668 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23278514938222616, + "mae": 0.12298233807086945, + "r2": 0.6538020968437195, + "score": -0.23278514938222616 + }, + "val": { + "rmse": 0.23353358071122673, + "mae": 0.1262781023979187, + "r2": 0.6357077360153198, + "score": -0.23353358071122673 + }, + "test": { + "rmse": 0.23738632838570875, + "mae": 0.15430940687656403, + "r2": 0.6149200201034546, + "score": -0.23738632838570875 + } + }, + "time": "0:01:52.324536", + "diversity": { + "train": { + "mean_centered_corr": -0.018679402768611908, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04577600955963135 + }, + "val": { + "mean_centered_corr": -0.019475068897008896, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05786792188882828 + }, + "test": { + "mean_centered_corr": -0.017192553728818893, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07846961915493011 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 21, + 25, + 29, + 28, + 0, + 21, + 1, + 25, + 21, + 0 + ], + "head_selection_time": "0:00:00.177379" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..31f233e53 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2247163600717624, + "mae": 0.12514503300189972, + "r2": 0.6773859262466431, + "score": -0.2247163600717624 + }, + "val": { + "rmse": 0.23828438459742213, + "mae": 0.13836915791034698, + "r2": 0.6207352876663208, + "score": -0.23828438459742213 + }, + "test": { + "rmse": 0.2530485190714714, + "mae": 0.17723064124584198, + "r2": 0.5624303817749023, + "score": -0.2530485190714714 + } + }, + "time": "0:02:02.636955", + "diversity": { + "train": { + "mean_centered_corr": -0.018269460648298264, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04634244740009308 + }, + "val": { + "mean_centered_corr": -0.018611595034599304, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06517667323350906 + }, + "test": { + "mean_centered_corr": -0.01696156896650791, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09637612849473953 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015773" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p001-evaluation/0.toml new file mode 100644 index 000000000..c0b709eb7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p001-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..5f27feefe --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p001-evaluation/0/report.json @@ -0,0 +1,654 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.21980748699677205, + "mae": 0.11748126894235611, + "r2": 0.6913268566131592, + "score": -0.21980748699677205 + }, + "val": { + "rmse": 0.2349816418639806, + "mae": 0.12830044329166412, + "r2": 0.6311760544776917, + "score": -0.2349816418639806 + }, + "test": { + "rmse": 0.24136844697035015, + "mae": 0.15835422277450562, + "r2": 0.6018922924995422, + "score": -0.24136844697035015 + } + }, + "time": "0:02:02.636955", + "diversity": { + "train": { + "mean_centered_corr": -0.018269460648298264, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04634244740009308 + }, + "val": { + "mean_centered_corr": -0.018611595034599304, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06517667323350906 + }, + "test": { + "mean_centered_corr": -0.01696156896650791, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09637612849473953 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..719e2a382 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,665 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22116132451653123, + "mae": 0.11670376360416412, + "r2": 0.6875127553939819, + "score": -0.22116132451653123 + }, + "val": { + "rmse": 0.23357213312617114, + "mae": 0.1273292750120163, + "r2": 0.6355874538421631, + "score": -0.23357213312617114 + }, + "test": { + "rmse": 0.23844724876452805, + "mae": 0.15294302999973297, + "r2": 0.6114702820777893, + "score": -0.23844724876452805 + } + }, + "time": "0:02:02.636955", + "diversity": { + "train": { + "mean_centered_corr": -0.018269460648298264, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04634244740009308 + }, + "val": { + "mean_centered_corr": -0.018611595034599304, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06517667323350906 + }, + "test": { + "mean_centered_corr": -0.01696156896650791, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09637612849473953 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 21, + 29, + 7, + 10, + 8, + 19, + 18 + ], + "head_selection_time": "0:00:00.134808" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..c663085b9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.23150610628324156, + "mae": 0.12310612946748734, + "r2": 0.6575959920883179, + "score": -0.23150610628324156 + }, + "val": { + "rmse": 0.239770922958097, + "mae": 0.1303773671388626, + "r2": 0.6159884333610535, + "score": -0.239770922958097 + }, + "test": { + "rmse": 0.23915230117801708, + "mae": 0.14451579749584198, + "r2": 0.6091692447662354, + "score": -0.23915230117801708 + } + }, + "time": "0:01:59.760644", + "diversity": { + "train": { + "mean_centered_corr": -0.018444906920194626, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04715638607740402 + }, + "val": { + "mean_centered_corr": -0.01882033608853817, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07111956179141998 + }, + "test": { + "mean_centered_corr": -0.016320303082466125, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09910114109516144 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 8 + ], + "head_selection_time": "0:00:00.015389" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p005-evaluation/0.toml new file mode 100644 index 000000000..932eafb54 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p005-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..f524ca666 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p005-evaluation/0/report.json @@ -0,0 +1,654 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22098960681519272, + "mae": 0.11940933763980865, + "r2": 0.6879978179931641, + "score": -0.22098960681519272 + }, + "val": { + "rmse": 0.23549047304076126, + "mae": 0.12960220873355865, + "r2": 0.6295770406723022, + "score": -0.23549047304076126 + }, + "test": { + "rmse": 0.24128326690664834, + "mae": 0.15859289467334747, + "r2": 0.6021732091903687, + "score": -0.24128326690664834 + } + }, + "time": "0:01:59.760644", + "diversity": { + "train": { + "mean_centered_corr": -0.018444906920194626, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04715638607740402 + }, + "val": { + "mean_centered_corr": -0.01882033608853817, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07111956179141998 + }, + "test": { + "mean_centered_corr": -0.016320303082466125, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09910114109516144 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..5fc0906bc --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,666 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22227666522397582, + "mae": 0.11813152581453323, + "r2": 0.6843529939651489, + "score": -0.22227666522397582 + }, + "val": { + "rmse": 0.23367947054972119, + "mae": 0.12708741426467896, + "r2": 0.6352524757385254, + "score": -0.23367947054972119 + }, + "test": { + "rmse": 0.23861273393383206, + "mae": 0.15416637063026428, + "r2": 0.6109308004379272, + "score": -0.23861273393383206 + } + }, + "time": "0:01:59.760644", + "diversity": { + "train": { + "mean_centered_corr": -0.018444906920194626, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04715638607740402 + }, + "val": { + "mean_centered_corr": -0.01882033608853817, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07111956179141998 + }, + "test": { + "mean_centered_corr": -0.016320303082466125, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09910114109516144 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 8, + 4, + 21, + 25, + 19, + 22, + 7, + 10, + 27 + ], + "head_selection_time": "0:00:00.144065" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..9e6107da0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23737684181165758, + "mae": 0.12491133064031601, + "r2": 0.640009880065918, + "score": -0.23737684181165758 + }, + "val": { + "rmse": 0.23850314174188256, + "mae": 0.13284452259540558, + "r2": 0.620038628578186, + "score": -0.23850314174188256 + }, + "test": { + "rmse": 0.23244720609452174, + "mae": 0.14553889632225037, + "r2": 0.6307774186134338, + "score": -0.23244720609452174 + } + }, + "time": "0:01:51.542856", + "diversity": { + "train": { + "mean_centered_corr": -0.018743393942713737, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045543067157268524 + }, + "val": { + "mean_centered_corr": -0.01886933483183384, + "mean_pairwise_disagreement": NaN, + "member_std": 0.060780204832553864 + }, + "test": { + "mean_centered_corr": -0.017534511163830757, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08134183287620544 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 28 + ], + "head_selection_time": "0:00:00.015619" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p01-evaluation/0.toml new file mode 100644 index 000000000..45c54b15b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p01-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..6eb74283d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p01-evaluation/0/report.json @@ -0,0 +1,654 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23111020547391228, + "mae": 0.12324945628643036, + "r2": 0.6587661504745483, + "score": -0.23111020547391228 + }, + "val": { + "rmse": 0.23543217967973734, + "mae": 0.1291837990283966, + "r2": 0.6297603845596313, + "score": -0.23543217967973734 + }, + "test": { + "rmse": 0.23786624351296862, + "mae": 0.1546480357646942, + "r2": 0.6133613586425781, + "score": -0.23786624351296862 + } + }, + "time": "0:01:51.542856", + "diversity": { + "train": { + "mean_centered_corr": -0.018743393942713737, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045543067157268524 + }, + "val": { + "mean_centered_corr": -0.01886933483183384, + "mean_pairwise_disagreement": NaN, + "member_std": 0.060780204832553864 + }, + "test": { + "mean_centered_corr": -0.017534511163830757, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08134183287620544 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..0aea97b14 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,665 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23300592784647, + "mae": 0.12180998921394348, + "r2": 0.6531450748443604, + "score": -0.23300592784647 + }, + "val": { + "rmse": 0.233651251716616, + "mae": 0.12775851786136627, + "r2": 0.6353405714035034, + "score": -0.233651251716616 + }, + "test": { + "rmse": 0.2427291888102559, + "mae": 0.16294866800308228, + "r2": 0.59739089012146, + "score": -0.2427291888102559 + } + }, + "time": "0:01:51.542856", + "diversity": { + "train": { + "mean_centered_corr": -0.018743393942713737, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045543067157268524 + }, + "val": { + "mean_centered_corr": -0.01886933483183384, + "mean_pairwise_disagreement": NaN, + "member_std": 0.060780204832553864 + }, + "test": { + "mean_centered_corr": -0.017534511163830757, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08134183287620544 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 28, + 21, + 25, + 6, + 1, + 4, + 25, + 10 + ], + "head_selection_time": "0:00:00.134347" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p02-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p02-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p02-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p02-best-head-evaluation/0/report.json new file mode 100644 index 000000000..41d7ee2c9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p02-best-head-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2396765489575744, + "mae": 0.126370370388031, + "r2": 0.633000910282135, + "score": -0.2396765489575744 + }, + "val": { + "rmse": 0.23843109390982442, + "mae": 0.13302145898342133, + "r2": 0.6202681064605713, + "score": -0.23843109390982442 + }, + "test": { + "rmse": 0.24318669684901736, + "mae": 0.16076454520225525, + "r2": 0.5958717465400696, + "score": -0.24318669684901736 + } + }, + "time": "0:01:50.591643", + "diversity": { + "train": { + "mean_centered_corr": -0.019360007718205452, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04535945504903793 + }, + "val": { + "mean_centered_corr": -0.01878437213599682, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05948599800467491 + }, + "test": { + "mean_centered_corr": -0.020098138600587845, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08049041777849197 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015508" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p02-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p02-evaluation/0.toml new file mode 100644 index 000000000..9be18618b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p02-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.02 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p02-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p02-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p02-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p02-evaluation/0/report.json new file mode 100644 index 000000000..84fa26804 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p02-evaluation/0/report.json @@ -0,0 +1,654 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2312604753596052, + "mae": 0.12086156755685806, + "r2": 0.6583222150802612, + "score": -0.2312604753596052 + }, + "val": { + "rmse": 0.23514095554484465, + "mae": 0.12670177221298218, + "r2": 0.6306757926940918, + "score": -0.23514095554484465 + }, + "test": { + "rmse": 0.23359667766611852, + "mae": 0.14610755443572998, + "r2": 0.6271167397499084, + "score": -0.23359667766611852 + } + }, + "time": "0:01:50.591643", + "diversity": { + "train": { + "mean_centered_corr": -0.019360007718205452, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04535945504903793 + }, + "val": { + "mean_centered_corr": -0.01878437213599682, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05948599800467491 + }, + "test": { + "mean_centered_corr": -0.020098138600587845, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08049041777849197 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p02-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p02-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p02-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p02-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..6c26dc9f8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p02-greedy-heads-evaluation/0/report.json @@ -0,0 +1,666 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23246381685745798, + "mae": 0.12184411287307739, + "r2": 0.6547572016716003, + "score": -0.23246381685745798 + }, + "val": { + "rmse": 0.2334736499346226, + "mae": 0.12559877336025238, + "r2": 0.6358946561813354, + "score": -0.2334736499346226 + }, + "test": { + "rmse": 0.23433651608184083, + "mae": 0.14739863574504852, + "r2": 0.6247509717941284, + "score": -0.23433651608184083 + } + }, + "time": "0:01:50.591643", + "diversity": { + "train": { + "mean_centered_corr": -0.019360007718205452, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04535945504903793 + }, + "val": { + "mean_centered_corr": -0.01878437213599682, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05948599800467491 + }, + "test": { + "mean_centered_corr": -0.020098138600587845, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08049041777849197 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 0, + 21, + 29, + 25, + 28, + 6, + 1, + 19 + ], + "head_selection_time": "0:00:00.146214" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..1725f1d2c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22576688139974624, + "mae": 0.12466424703598022, + "r2": 0.6743625402450562, + "score": -0.22576688139974624 + }, + "val": { + "rmse": 0.2388823307131178, + "mae": 0.1381048709154129, + "r2": 0.6188294887542725, + "score": -0.2388823307131178 + }, + "test": { + "rmse": 0.25624351202497053, + "mae": 0.18258099257946014, + "r2": 0.5513110756874084, + "score": -0.25624351202497053 + } + }, + "time": "0:02:01.596363", + "diversity": { + "train": { + "mean_centered_corr": -0.019050154834985733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046319905668497086 + }, + "val": { + "mean_centered_corr": -0.018872274085879326, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06369400024414062 + }, + "test": { + "mean_centered_corr": -0.016154184937477112, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0944347083568573 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015532" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p001-evaluation/0.toml new file mode 100644 index 000000000..c12e9e918 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p001-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..96fc04b8d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p001-evaluation/0/report.json @@ -0,0 +1,654 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2212921485786695, + "mae": 0.11681316047906876, + "r2": 0.6871429681777954, + "score": -0.2212921485786695 + }, + "val": { + "rmse": 0.23532450983472006, + "mae": 0.12748965620994568, + "r2": 0.6300989389419556, + "score": -0.23532450983472006 + }, + "test": { + "rmse": 0.2376596833156001, + "mae": 0.15176545083522797, + "r2": 0.6140326261520386, + "score": -0.2376596833156001 + } + }, + "time": "0:02:01.596363", + "diversity": { + "train": { + "mean_centered_corr": -0.019050154834985733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046319905668497086 + }, + "val": { + "mean_centered_corr": -0.018872274085879326, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06369400024414062 + }, + "test": { + "mean_centered_corr": -0.016154184937477112, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0944347083568573 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..96d88dd1b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,663 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22446870130378824, + "mae": 0.1184302344918251, + "r2": 0.6780966520309448, + "score": -0.22446870130378824 + }, + "val": { + "rmse": 0.23408501487488748, + "mae": 0.126699760556221, + "r2": 0.6339853405952454, + "score": -0.23408501487488748 + }, + "test": { + "rmse": 0.23576018763280351, + "mae": 0.14970247447490692, + "r2": 0.6201776266098022, + "score": -0.23576018763280351 + } + }, + "time": "0:02:01.596363", + "diversity": { + "train": { + "mean_centered_corr": -0.019050154834985733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046319905668497086 + }, + "val": { + "mean_centered_corr": -0.018872274085879326, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06369400024414062 + }, + "test": { + "mean_centered_corr": -0.016154184937477112, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0944347083568573 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 8, + 21, + 29, + 6, + 22 + ], + "head_selection_time": "0:00:00.102858" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..6db4b89df --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22990253367135435, + "mae": 0.12059187889099121, + "r2": 0.6623230576515198, + "score": -0.22990253367135435 + }, + "val": { + "rmse": 0.23844761590759667, + "mae": 0.1282438039779663, + "r2": 0.6202155351638794, + "score": -0.23844761590759667 + }, + "test": { + "rmse": 0.2416540932193429, + "mae": 0.14754453301429749, + "r2": 0.6009494066238403, + "score": -0.2416540932193429 + } + }, + "time": "0:02:02.101286", + "diversity": { + "train": { + "mean_centered_corr": -0.018734341487288475, + "mean_pairwise_disagreement": NaN, + "member_std": 0.044835660606622696 + }, + "val": { + "mean_centered_corr": -0.018935222178697586, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06088249757885933 + }, + "test": { + "mean_centered_corr": -0.01582489162683487, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08836037665605545 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 8 + ], + "head_selection_time": "0:00:00.015062" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p005-evaluation/0.toml new file mode 100644 index 000000000..7de9c7b34 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p005-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..450021870 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p005-evaluation/0/report.json @@ -0,0 +1,654 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2216062550188277, + "mae": 0.1181904599070549, + "r2": 0.6862542033195496, + "score": -0.2216062550188277 + }, + "val": { + "rmse": 0.23472067454979642, + "mae": 0.12802676856517792, + "r2": 0.6319948434829712, + "score": -0.23472067454979642 + }, + "test": { + "rmse": 0.23644700488851306, + "mae": 0.15162619948387146, + "r2": 0.6179614067077637, + "score": -0.23644700488851306 + } + }, + "time": "0:02:02.101286", + "diversity": { + "train": { + "mean_centered_corr": -0.018734341487288475, + "mean_pairwise_disagreement": NaN, + "member_std": 0.044835660606622696 + }, + "val": { + "mean_centered_corr": -0.018935222178697586, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06088249757885933 + }, + "test": { + "mean_centered_corr": -0.01582489162683487, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08836037665605545 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..d21ed4c35 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,664 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2225587447309463, + "mae": 0.11954773962497711, + "r2": 0.6835513710975647, + "score": -0.2225587447309463 + }, + "val": { + "rmse": 0.23314779382394482, + "mae": 0.12646809220314026, + "r2": 0.6369103193283081, + "score": -0.23314779382394482 + }, + "test": { + "rmse": 0.23448866631035667, + "mae": 0.14718787372112274, + "r2": 0.6242635846138, + "score": -0.23448866631035667 + } + }, + "time": "0:02:02.101286", + "diversity": { + "train": { + "mean_centered_corr": -0.018734341487288475, + "mean_pairwise_disagreement": NaN, + "member_std": 0.044835660606622696 + }, + "val": { + "mean_centered_corr": -0.018935222178697586, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06088249757885933 + }, + "test": { + "mean_centered_corr": -0.01582489162683487, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08836037665605545 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 8, + 25, + 19, + 4, + 21, + 22, + 5 + ], + "head_selection_time": "0:00:00.116994" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..e9aadf54d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23604626832625122, + "mae": 0.13103090226650238, + "r2": 0.6440342664718628, + "score": -0.23604626832625122 + }, + "val": { + "rmse": 0.2383569065742623, + "mae": 0.13819104433059692, + "r2": 0.6205043792724609, + "score": -0.2383569065742623 + }, + "test": { + "rmse": 0.244620972160157, + "mae": 0.16388729214668274, + "r2": 0.5910906791687012, + "score": -0.244620972160157 + } + }, + "time": "0:01:52.165374", + "diversity": { + "train": { + "mean_centered_corr": -0.018348872661590576, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04553309082984924 + }, + "val": { + "mean_centered_corr": -0.019239172339439392, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05902950465679169 + }, + "test": { + "mean_centered_corr": -0.017977142706513405, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08112667500972748 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 16 + ], + "head_selection_time": "0:00:00.015406" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p01-evaluation/0.toml new file mode 100644 index 000000000..0d76249b7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p01-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..bbac15ab2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p01-evaluation/0/report.json @@ -0,0 +1,654 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23063516266200654, + "mae": 0.12075714766979218, + "r2": 0.6601674556732178, + "score": -0.23063516266200654 + }, + "val": { + "rmse": 0.23505473908329258, + "mae": 0.12712784111499786, + "r2": 0.6309465169906616, + "score": -0.23505473908329258 + }, + "test": { + "rmse": 0.23281250512039894, + "mae": 0.14543399214744568, + "r2": 0.6296160221099854, + "score": -0.23281250512039894 + } + }, + "time": "0:01:52.165374", + "diversity": { + "train": { + "mean_centered_corr": -0.018348872661590576, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04553309082984924 + }, + "val": { + "mean_centered_corr": -0.019239172339439392, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05902950465679169 + }, + "test": { + "mean_centered_corr": -0.017977142706513405, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08112667500972748 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..700225951 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,666 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23284740122388672, + "mae": 0.1222502812743187, + "r2": 0.6536169052124023, + "score": -0.23284740122388672 + }, + "val": { + "rmse": 0.2334130414280393, + "mae": 0.12650004029273987, + "r2": 0.636083722114563, + "score": -0.2334130414280393 + }, + "test": { + "rmse": 0.23649611689361716, + "mae": 0.15280967950820923, + "r2": 0.6178027391433716, + "score": -0.23649611689361716 + } + }, + "time": "0:01:52.165374", + "diversity": { + "train": { + "mean_centered_corr": -0.018348872661590576, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04553309082984924 + }, + "val": { + "mean_centered_corr": -0.019239172339439392, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05902950465679169 + }, + "test": { + "mean_centered_corr": -0.017977142706513405, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08112667500972748 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 16, + 1, + 10, + 25, + 21, + 28, + 19, + 0, + 1 + ], + "head_selection_time": "0:00:00.146443" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p02-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p02-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p02-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p02-best-head-evaluation/0/report.json new file mode 100644 index 000000000..fcf2bb9e1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p02-best-head-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23952544677482882, + "mae": 0.12448744475841522, + "r2": 0.6334635019302368, + "score": -0.23952544677482882 + }, + "val": { + "rmse": 0.23747199640064248, + "mae": 0.13178245723247528, + "r2": 0.6233169436454773, + "score": -0.23747199640064248 + }, + "test": { + "rmse": 0.24725484176674034, + "mae": 0.16568316519260406, + "r2": 0.582237720489502, + "score": -0.24725484176674034 + } + }, + "time": "0:01:53.326611", + "diversity": { + "train": { + "mean_centered_corr": -0.018683521077036858, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04501955956220627 + }, + "val": { + "mean_centered_corr": -0.019395897164940834, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05899025872349739 + }, + "test": { + "mean_centered_corr": -0.017709530889987946, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0825389176607132 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015209" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p02-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p02-evaluation/0.toml new file mode 100644 index 000000000..e17dd27d1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p02-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.02 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p02-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p02-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p02-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p02-evaluation/0/report.json new file mode 100644 index 000000000..7982459ce --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p02-evaluation/0/report.json @@ -0,0 +1,654 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23193801072582776, + "mae": 0.1212281584739685, + "r2": 0.6563172340393066, + "score": -0.23193801072582776 + }, + "val": { + "rmse": 0.23511142271763394, + "mae": 0.12687157094478607, + "r2": 0.6307685375213623, + "score": -0.23511142271763394 + }, + "test": { + "rmse": 0.23473994135203347, + "mae": 0.14816908538341522, + "r2": 0.6234579086303711, + "score": -0.23473994135203347 + } + }, + "time": "0:01:53.326611", + "diversity": { + "train": { + "mean_centered_corr": -0.018683521077036858, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04501955956220627 + }, + "val": { + "mean_centered_corr": -0.019395897164940834, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05899025872349739 + }, + "test": { + "mean_centered_corr": -0.017709530889987946, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0825389176607132 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p02-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p02-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p02-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p02-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..192aaf7ed --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p02-greedy-heads-evaluation/0/report.json @@ -0,0 +1,665 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23442632590013812, + "mae": 0.1224464401602745, + "r2": 0.6489033699035645, + "score": -0.23442632590013812 + }, + "val": { + "rmse": 0.23366474775887086, + "mae": 0.12645675241947174, + "r2": 0.6352983713150024, + "score": -0.23366474775887086 + }, + "test": { + "rmse": 0.23646780882908092, + "mae": 0.15090638399124146, + "r2": 0.617894172668457, + "score": -0.23646780882908092 + } + }, + "time": "0:01:53.326611", + "diversity": { + "train": { + "mean_centered_corr": -0.018683521077036858, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04501955956220627 + }, + "val": { + "mean_centered_corr": -0.019395897164940834, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05899025872349739 + }, + "test": { + "mean_centered_corr": -0.017709530889987946, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0825389176607132 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 29, + 25, + 10, + 1, + 0, + 19, + 28 + ], + "head_selection_time": "0:00:00.130044" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p00025-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p00025-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p00025-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p00025-best-head-evaluation/0/report.json new file mode 100644 index 000000000..69b541d02 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p00025-best-head-evaluation/0/report.json @@ -0,0 +1,207 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23782740036230163, + "mae": 0.13576887547969818, + "r2": 0.6386420130729675, + "score": -0.23782740036230163 + }, + "val": { + "rmse": 0.23857661998288315, + "mae": 0.13661375641822815, + "r2": 0.6198044419288635, + "score": -0.23857661998288315 + }, + "test": { + "rmse": 0.2544202590204169, + "mae": 0.17306958138942719, + "r2": 0.557673454284668, + "score": -0.2544202590204169 + } + }, + "time": "0:01:21.016540", + "diversity": { + "train": { + "mean_centered_corr": -0.01925019733607769, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04815442115068436 + }, + "val": { + "mean_centered_corr": -0.018709633499383926, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06300598382949829 + }, + "test": { + "mean_centered_corr": -0.018054556101560593, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08491437882184982 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0 + ], + "head_selection_time": "0:00:00.015229" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p00025-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p00025-evaluation/0.toml new file mode 100644 index 000000000..e8b03a128 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p00025-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.00025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p00025-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p00025-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p00025-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p00025-evaluation/0/report.json new file mode 100644 index 000000000..4e423b1d6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p00025-evaluation/0/report.json @@ -0,0 +1,203 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23108709757774937, + "mae": 0.1223861500620842, + "r2": 0.6588343381881714, + "score": -0.23108709757774937 + }, + "val": { + "rmse": 0.23484986994593354, + "mae": 0.12684030830860138, + "r2": 0.6315895318984985, + "score": -0.23484986994593354 + }, + "test": { + "rmse": 0.23462197483665095, + "mae": 0.1482354700565338, + "r2": 0.6238362193107605, + "score": -0.23462197483665095 + } + }, + "time": "0:01:21.016540", + "diversity": { + "train": { + "mean_centered_corr": -0.01925019733607769, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04815442115068436 + }, + "val": { + "mean_centered_corr": -0.018709633499383926, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06300598382949829 + }, + "test": { + "mean_centered_corr": -0.018054556101560593, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08491437882184982 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p00025-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p00025-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p00025-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p00025-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..dae752a86 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p00025-greedy-heads-evaluation/0/report.json @@ -0,0 +1,217 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2319917866834578, + "mae": 0.12356825172901154, + "r2": 0.6561578512191772, + "score": -0.2319917866834578 + }, + "val": { + "rmse": 0.23320292020753594, + "mae": 0.12550441920757294, + "r2": 0.636738657951355, + "score": -0.23320292020753594 + }, + "test": { + "rmse": 0.23031042609193297, + "mae": 0.14081889390945435, + "r2": 0.6375343799591064, + "score": -0.23031042609193297 + } + }, + "time": "0:01:21.016540", + "diversity": { + "train": { + "mean_centered_corr": -0.01925019733607769, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04815442115068436 + }, + "val": { + "mean_centered_corr": -0.018709633499383926, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06300598382949829 + }, + "test": { + "mean_centered_corr": -0.018054556101560593, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08491437882184982 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0, + 4, + 28, + 1, + 19, + 24, + 16, + 4, + 1, + 20, + 21 + ], + "head_selection_time": "0:00:00.174232" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..3d8fb6589 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0005-best-head-evaluation/0/report.json @@ -0,0 +1,207 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2320922951078084, + "mae": 0.12688623368740082, + "r2": 0.6558598279953003, + "score": -0.2320922951078084 + }, + "val": { + "rmse": 0.2390755017179478, + "mae": 0.13819952309131622, + "r2": 0.6182127594947815, + "score": -0.2390755017179478 + }, + "test": { + "rmse": 0.25168892959044736, + "mae": 0.1720852106809616, + "r2": 0.5671197175979614, + "score": -0.25168892959044736 + } + }, + "time": "0:01:27.443113", + "diversity": { + "train": { + "mean_centered_corr": -0.018731974065303802, + "mean_pairwise_disagreement": NaN, + "member_std": 0.049796536564826965 + }, + "val": { + "mean_centered_corr": -0.019370803609490395, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06845120340585709 + }, + "test": { + "mean_centered_corr": -0.016793658956885338, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0947653204202652 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 22 + ], + "head_selection_time": "0:00:00.015348" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0005-evaluation/0.toml new file mode 100644 index 000000000..e87934418 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0005-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.0005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0005-evaluation/0/report.json new file mode 100644 index 000000000..45713e42c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0005-evaluation/0/report.json @@ -0,0 +1,203 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2222445177940426, + "mae": 0.12026011943817139, + "r2": 0.6844443082809448, + "score": -0.2222445177940426 + }, + "val": { + "rmse": 0.23510992538054096, + "mae": 0.12966062128543854, + "r2": 0.6307731866836548, + "score": -0.23510992538054096 + }, + "test": { + "rmse": 0.24175341990615093, + "mae": 0.1596364676952362, + "r2": 0.6006213426589966, + "score": -0.24175341990615093 + } + }, + "time": "0:01:27.443113", + "diversity": { + "train": { + "mean_centered_corr": -0.018731974065303802, + "mean_pairwise_disagreement": NaN, + "member_std": 0.049796536564826965 + }, + "val": { + "mean_centered_corr": -0.019370803609490395, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06845120340585709 + }, + "test": { + "mean_centered_corr": -0.016793658956885338, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0947653204202652 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..aaca2bced --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22296470056985443, + "mae": 0.12100711464881897, + "r2": 0.682395875453949, + "score": -0.22296470056985443 + }, + "val": { + "rmse": 0.2333487134502395, + "mae": 0.12775763869285583, + "r2": 0.6362842321395874, + "score": -0.2333487134502395 + }, + "test": { + "rmse": 0.2416932153437863, + "mae": 0.16084370017051697, + "r2": 0.6008202433586121, + "score": -0.2416932153437863 + } + }, + "time": "0:01:27.443113", + "diversity": { + "train": { + "mean_centered_corr": -0.018731974065303802, + "mean_pairwise_disagreement": NaN, + "member_std": 0.049796536564826965 + }, + "val": { + "mean_centered_corr": -0.019370803609490395, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06845120340585709 + }, + "test": { + "mean_centered_corr": -0.016793658956885338, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0947653204202652 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 22, + 3, + 4, + 8, + 25, + 20, + 21 + ], + "head_selection_time": "0:00:00.116276" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..01cc37386 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p001-best-head-evaluation/0/report.json @@ -0,0 +1,207 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2255253318194067, + "mae": 0.1253618746995926, + "r2": 0.6750589609146118, + "score": -0.2255253318194067 + }, + "val": { + "rmse": 0.23949031043262045, + "mae": 0.13341857492923737, + "r2": 0.6168867349624634, + "score": -0.23949031043262045 + }, + "test": { + "rmse": 0.24155888192007524, + "mae": 0.14752615988254547, + "r2": 0.6012638807296753, + "score": -0.24155888192007524 + } + }, + "time": "0:01:30.844410", + "diversity": { + "train": { + "mean_centered_corr": -0.017563171684741974, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048281844705343246 + }, + "val": { + "mean_centered_corr": -0.01754843257367611, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07056254893541336 + }, + "test": { + "mean_centered_corr": -0.016294721513986588, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09666775166988373 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19 + ], + "head_selection_time": "0:00:00.015134" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p001-evaluation/0.toml new file mode 100644 index 000000000..b3d60a0de --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p001-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.001 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p001-evaluation/0/report.json new file mode 100644 index 000000000..c14e16500 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p001-evaluation/0/report.json @@ -0,0 +1,203 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2197185345348766, + "mae": 0.11797432601451874, + "r2": 0.6915766000747681, + "score": -0.2197185345348766 + }, + "val": { + "rmse": 0.23467774706155475, + "mae": 0.1267256885766983, + "r2": 0.632129430770874, + "score": -0.23467774706155475 + }, + "test": { + "rmse": 0.23716614057842658, + "mae": 0.1508055180311203, + "r2": 0.6156339645385742, + "score": -0.23716614057842658 + } + }, + "time": "0:01:30.844410", + "diversity": { + "train": { + "mean_centered_corr": -0.017563171684741974, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048281844705343246 + }, + "val": { + "mean_centered_corr": -0.01754843257367611, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07056254893541336 + }, + "test": { + "mean_centered_corr": -0.016294721513986588, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09666775166988373 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..f6a84f3d5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,219 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.22057000931073897, + "mae": 0.11852128803730011, + "r2": 0.6891815662384033, + "score": -0.22057000931073897 + }, + "val": { + "rmse": 0.23316146280351166, + "mae": 0.12596334517002106, + "r2": 0.6368677616119385, + "score": -0.23316146280351166 + }, + "test": { + "rmse": 0.23452524296935331, + "mae": 0.14668044447898865, + "r2": 0.6241463422775269, + "score": -0.23452524296935331 + } + }, + "time": "0:01:30.844410", + "diversity": { + "train": { + "mean_centered_corr": -0.017563171684741974, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048281844705343246 + }, + "val": { + "mean_centered_corr": -0.01754843257367611, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07056254893541336 + }, + "test": { + "mean_centered_corr": -0.016294721513986588, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09666775166988373 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19, + 4, + 16, + 12, + 0, + 25, + 31, + 3, + 22, + 29, + 12, + 27, + 20 + ], + "head_selection_time": "0:00:00.204011" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0025-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0025-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0025-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0025-best-head-evaluation/0/report.json new file mode 100644 index 000000000..bed297dff --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0025-best-head-evaluation/0/report.json @@ -0,0 +1,207 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2261742520907439, + "mae": 0.12224072217941284, + "r2": 0.6731863021850586, + "score": -0.2261742520907439 + }, + "val": { + "rmse": 0.2377136850270048, + "mae": 0.13327811658382416, + "r2": 0.622549831867218, + "score": -0.2377136850270048 + }, + "test": { + "rmse": 0.262345673404661, + "mae": 0.17990553379058838, + "r2": 0.5296865105628967, + "score": -0.262345673404661 + } + }, + "time": "0:01:26.676984", + "diversity": { + "train": { + "mean_centered_corr": -0.01859835349023342, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04920431599020958 + }, + "val": { + "mean_centered_corr": -0.019710257649421692, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06811030209064484 + }, + "test": { + "mean_centered_corr": -0.013359437696635723, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09920918941497803 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015248" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0025-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0025-evaluation/0.toml new file mode 100644 index 000000000..8d6397619 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0025-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.0025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0025-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0025-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0025-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0025-evaluation/0/report.json new file mode 100644 index 000000000..0ce8dd477 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0025-evaluation/0/report.json @@ -0,0 +1,203 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22265827445732783, + "mae": 0.11788544058799744, + "r2": 0.6832683086395264, + "score": -0.22265827445732783 + }, + "val": { + "rmse": 0.23474623366768377, + "mae": 0.1270996779203415, + "r2": 0.6319146752357483, + "score": -0.23474623366768377 + }, + "test": { + "rmse": 0.23936563514587833, + "mae": 0.15535929799079895, + "r2": 0.6084716320037842, + "score": -0.23936563514587833 + } + }, + "time": "0:01:26.676984", + "diversity": { + "train": { + "mean_centered_corr": -0.01859835349023342, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04920431599020958 + }, + "val": { + "mean_centered_corr": -0.019710257649421692, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06811030209064484 + }, + "test": { + "mean_centered_corr": -0.013359437696635723, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09920918941497803 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0025-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0025-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0025-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0025-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..bc918699d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0025-greedy-heads-evaluation/0/report.json @@ -0,0 +1,214 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22450348389711117, + "mae": 0.11922673135995865, + "r2": 0.6779968738555908, + "score": -0.22450348389711117 + }, + "val": { + "rmse": 0.23295802298295287, + "mae": 0.1258699893951416, + "r2": 0.6375011801719666, + "score": -0.23295802298295287 + }, + "test": { + "rmse": 0.23903366020873654, + "mae": 0.15511320531368256, + "r2": 0.6095569133758545, + "score": -0.23903366020873654 + } + }, + "time": "0:01:26.676984", + "diversity": { + "train": { + "mean_centered_corr": -0.01859835349023342, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04920431599020958 + }, + "val": { + "mean_centered_corr": -0.019710257649421692, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06811030209064484 + }, + "test": { + "mean_centered_corr": -0.013359437696635723, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09920918941497803 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 22, + 12, + 16, + 21, + 19, + 25, + 29 + ], + "head_selection_time": "0:00:00.129748" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p00025-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p00025-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p00025-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p00025-best-head-evaluation/0/report.json new file mode 100644 index 000000000..c6c0cd90a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p00025-best-head-evaluation/0/report.json @@ -0,0 +1,207 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0p0001_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23782740036230163, + "mae": 0.13576887547969818, + "r2": 0.6386420130729675, + "score": -0.23782740036230163 + }, + "val": { + "rmse": 0.23857661998288315, + "mae": 0.13661375641822815, + "r2": 0.6198044419288635, + "score": -0.23857661998288315 + }, + "test": { + "rmse": 0.2544202590204169, + "mae": 0.17306958138942719, + "r2": 0.557673454284668, + "score": -0.2544202590204169 + } + }, + "time": "0:01:20.336729", + "diversity": { + "train": { + "mean_centered_corr": -0.01925019733607769, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04815442115068436 + }, + "val": { + "mean_centered_corr": -0.018709633499383926, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06300598382949829 + }, + "test": { + "mean_centered_corr": -0.018054556101560593, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08491437882184982 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0 + ], + "head_selection_time": "0:00:00.015352" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p00025-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p00025-evaluation/0.toml new file mode 100644 index 000000000..5fdf43534 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p00025-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.00025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p00025-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p00025-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p00025-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p00025-evaluation/0/report.json new file mode 100644 index 000000000..e5dff772e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p00025-evaluation/0/report.json @@ -0,0 +1,203 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0p0001_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23108709757774937, + "mae": 0.1223861500620842, + "r2": 0.6588343381881714, + "score": -0.23108709757774937 + }, + "val": { + "rmse": 0.23484986994593354, + "mae": 0.12684030830860138, + "r2": 0.6315895318984985, + "score": -0.23484986994593354 + }, + "test": { + "rmse": 0.23462197483665095, + "mae": 0.1482354700565338, + "r2": 0.6238362193107605, + "score": -0.23462197483665095 + } + }, + "time": "0:01:20.336729", + "diversity": { + "train": { + "mean_centered_corr": -0.01925019733607769, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04815442115068436 + }, + "val": { + "mean_centered_corr": -0.018709633499383926, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06300598382949829 + }, + "test": { + "mean_centered_corr": -0.018054556101560593, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08491437882184982 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p00025-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p00025-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p00025-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p00025-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..d5837e7f3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p00025-greedy-heads-evaluation/0/report.json @@ -0,0 +1,217 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0p0001_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2319917866834578, + "mae": 0.12356825172901154, + "r2": 0.6561578512191772, + "score": -0.2319917866834578 + }, + "val": { + "rmse": 0.23320292020753594, + "mae": 0.12550441920757294, + "r2": 0.636738657951355, + "score": -0.23320292020753594 + }, + "test": { + "rmse": 0.23031042609193297, + "mae": 0.14081889390945435, + "r2": 0.6375343799591064, + "score": -0.23031042609193297 + } + }, + "time": "0:01:20.336729", + "diversity": { + "train": { + "mean_centered_corr": -0.01925019733607769, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04815442115068436 + }, + "val": { + "mean_centered_corr": -0.018709633499383926, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06300598382949829 + }, + "test": { + "mean_centered_corr": -0.018054556101560593, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08491437882184982 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0, + 4, + 28, + 1, + 19, + 24, + 16, + 4, + 1, + 20, + 21 + ], + "head_selection_time": "0:00:00.174065" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..0efc6a4da --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0005-best-head-evaluation/0/report.json @@ -0,0 +1,207 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0p0001_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2320922951078084, + "mae": 0.12688623368740082, + "r2": 0.6558598279953003, + "score": -0.2320922951078084 + }, + "val": { + "rmse": 0.2390755017179478, + "mae": 0.13819952309131622, + "r2": 0.6182127594947815, + "score": -0.2390755017179478 + }, + "test": { + "rmse": 0.25168892959044736, + "mae": 0.1720852106809616, + "r2": 0.5671197175979614, + "score": -0.25168892959044736 + } + }, + "time": "0:01:27.032752", + "diversity": { + "train": { + "mean_centered_corr": -0.018731974065303802, + "mean_pairwise_disagreement": NaN, + "member_std": 0.049796536564826965 + }, + "val": { + "mean_centered_corr": -0.019370803609490395, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06845120340585709 + }, + "test": { + "mean_centered_corr": -0.016793658956885338, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0947653204202652 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 22 + ], + "head_selection_time": "0:00:00.015405" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0005-evaluation/0.toml new file mode 100644 index 000000000..a2f9fafcb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0005-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.0005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0005-evaluation/0/report.json new file mode 100644 index 000000000..2ab000289 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0005-evaluation/0/report.json @@ -0,0 +1,203 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0p0001_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2222445177940426, + "mae": 0.12026011943817139, + "r2": 0.6844443082809448, + "score": -0.2222445177940426 + }, + "val": { + "rmse": 0.23510992538054096, + "mae": 0.12966062128543854, + "r2": 0.6307731866836548, + "score": -0.23510992538054096 + }, + "test": { + "rmse": 0.24175341990615093, + "mae": 0.1596364676952362, + "r2": 0.6006213426589966, + "score": -0.24175341990615093 + } + }, + "time": "0:01:27.032752", + "diversity": { + "train": { + "mean_centered_corr": -0.018731974065303802, + "mean_pairwise_disagreement": NaN, + "member_std": 0.049796536564826965 + }, + "val": { + "mean_centered_corr": -0.019370803609490395, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06845120340585709 + }, + "test": { + "mean_centered_corr": -0.016793658956885338, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0947653204202652 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..93707571d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0p0001_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22296470056985443, + "mae": 0.12100711464881897, + "r2": 0.682395875453949, + "score": -0.22296470056985443 + }, + "val": { + "rmse": 0.2333487134502395, + "mae": 0.12775763869285583, + "r2": 0.6362842321395874, + "score": -0.2333487134502395 + }, + "test": { + "rmse": 0.2416932153437863, + "mae": 0.16084370017051697, + "r2": 0.6008202433586121, + "score": -0.2416932153437863 + } + }, + "time": "0:01:27.032752", + "diversity": { + "train": { + "mean_centered_corr": -0.018731974065303802, + "mean_pairwise_disagreement": NaN, + "member_std": 0.049796536564826965 + }, + "val": { + "mean_centered_corr": -0.019370803609490395, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06845120340585709 + }, + "test": { + "mean_centered_corr": -0.016793658956885338, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0947653204202652 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 22, + 3, + 4, + 8, + 25, + 20, + 21 + ], + "head_selection_time": "0:00:00.116690" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..ba45263d6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p001-best-head-evaluation/0/report.json @@ -0,0 +1,207 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0p0001_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2255253318194067, + "mae": 0.1253618746995926, + "r2": 0.6750589609146118, + "score": -0.2255253318194067 + }, + "val": { + "rmse": 0.23949031043262045, + "mae": 0.13341857492923737, + "r2": 0.6168867349624634, + "score": -0.23949031043262045 + }, + "test": { + "rmse": 0.24155888192007524, + "mae": 0.14752615988254547, + "r2": 0.6012638807296753, + "score": -0.24155888192007524 + } + }, + "time": "0:01:29.541702", + "diversity": { + "train": { + "mean_centered_corr": -0.017563171684741974, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048281844705343246 + }, + "val": { + "mean_centered_corr": -0.01754843257367611, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07056254893541336 + }, + "test": { + "mean_centered_corr": -0.016294721513986588, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09666775166988373 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19 + ], + "head_selection_time": "0:00:00.015004" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p001-evaluation/0.toml new file mode 100644 index 000000000..3ad7b4def --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p001-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.001 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p001-evaluation/0/report.json new file mode 100644 index 000000000..51b5864bf --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p001-evaluation/0/report.json @@ -0,0 +1,203 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0p0001_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2197185345348766, + "mae": 0.11797432601451874, + "r2": 0.6915766000747681, + "score": -0.2197185345348766 + }, + "val": { + "rmse": 0.23467774706155475, + "mae": 0.1267256885766983, + "r2": 0.632129430770874, + "score": -0.23467774706155475 + }, + "test": { + "rmse": 0.23716614057842658, + "mae": 0.1508055180311203, + "r2": 0.6156339645385742, + "score": -0.23716614057842658 + } + }, + "time": "0:01:29.541702", + "diversity": { + "train": { + "mean_centered_corr": -0.017563171684741974, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048281844705343246 + }, + "val": { + "mean_centered_corr": -0.01754843257367611, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07056254893541336 + }, + "test": { + "mean_centered_corr": -0.016294721513986588, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09666775166988373 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..4a0a5ed75 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,219 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0p0001_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.22057000931073897, + "mae": 0.11852128803730011, + "r2": 0.6891815662384033, + "score": -0.22057000931073897 + }, + "val": { + "rmse": 0.23316146280351166, + "mae": 0.12596334517002106, + "r2": 0.6368677616119385, + "score": -0.23316146280351166 + }, + "test": { + "rmse": 0.23452524296935331, + "mae": 0.14668044447898865, + "r2": 0.6241463422775269, + "score": -0.23452524296935331 + } + }, + "time": "0:01:29.541702", + "diversity": { + "train": { + "mean_centered_corr": -0.017563171684741974, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048281844705343246 + }, + "val": { + "mean_centered_corr": -0.01754843257367611, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07056254893541336 + }, + "test": { + "mean_centered_corr": -0.016294721513986588, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09666775166988373 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19, + 4, + 16, + 12, + 0, + 25, + 31, + 3, + 22, + 29, + 12, + 27, + 20 + ], + "head_selection_time": "0:00:00.203091" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0025-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0025-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0025-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0025-best-head-evaluation/0/report.json new file mode 100644 index 000000000..bbb2b37fa --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0025-best-head-evaluation/0/report.json @@ -0,0 +1,207 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0p0001_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2261742520907439, + "mae": 0.12224072217941284, + "r2": 0.6731863021850586, + "score": -0.2261742520907439 + }, + "val": { + "rmse": 0.2377136850270048, + "mae": 0.13327811658382416, + "r2": 0.622549831867218, + "score": -0.2377136850270048 + }, + "test": { + "rmse": 0.262345673404661, + "mae": 0.17990553379058838, + "r2": 0.5296865105628967, + "score": -0.262345673404661 + } + }, + "time": "0:01:27.105568", + "diversity": { + "train": { + "mean_centered_corr": -0.01859835349023342, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04920431599020958 + }, + "val": { + "mean_centered_corr": -0.019710257649421692, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06811030209064484 + }, + "test": { + "mean_centered_corr": -0.013359437696635723, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09920918941497803 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015126" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0025-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0025-evaluation/0.toml new file mode 100644 index 000000000..d9ba67e38 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0025-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.0025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0025-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0025-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0025-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0025-evaluation/0/report.json new file mode 100644 index 000000000..fca4bfe7b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0025-evaluation/0/report.json @@ -0,0 +1,203 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0p0001_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22265827445732783, + "mae": 0.11788544058799744, + "r2": 0.6832683086395264, + "score": -0.22265827445732783 + }, + "val": { + "rmse": 0.23474623366768377, + "mae": 0.1270996779203415, + "r2": 0.6319146752357483, + "score": -0.23474623366768377 + }, + "test": { + "rmse": 0.23936563514587833, + "mae": 0.15535929799079895, + "r2": 0.6084716320037842, + "score": -0.23936563514587833 + } + }, + "time": "0:01:27.105568", + "diversity": { + "train": { + "mean_centered_corr": -0.01859835349023342, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04920431599020958 + }, + "val": { + "mean_centered_corr": -0.019710257649421692, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06811030209064484 + }, + "test": { + "mean_centered_corr": -0.013359437696635723, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09920918941497803 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0025-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0025-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0025-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0025-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..81108d9f0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0025-greedy-heads-evaluation/0/report.json @@ -0,0 +1,214 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0p0001_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22450348389711117, + "mae": 0.11922673135995865, + "r2": 0.6779968738555908, + "score": -0.22450348389711117 + }, + "val": { + "rmse": 0.23295802298295287, + "mae": 0.1258699893951416, + "r2": 0.6375011801719666, + "score": -0.23295802298295287 + }, + "test": { + "rmse": 0.23903366020873654, + "mae": 0.15511320531368256, + "r2": 0.6095569133758545, + "score": -0.23903366020873654 + } + }, + "time": "0:01:27.105568", + "diversity": { + "train": { + "mean_centered_corr": -0.01859835349023342, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04920431599020958 + }, + "val": { + "mean_centered_corr": -0.019710257649421692, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06811030209064484 + }, + "test": { + "mean_centered_corr": -0.013359437696635723, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09920918941497803 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 22, + 12, + 16, + 21, + 19, + 25, + 29 + ], + "head_selection_time": "0:00:00.129037" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p00025-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p00025-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p00025-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p00025-best-head-evaluation/0/report.json new file mode 100644 index 000000000..b8acbff5c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p00025-best-head-evaluation/0/report.json @@ -0,0 +1,207 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0p001_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23782740036230163, + "mae": 0.13576887547969818, + "r2": 0.6386420130729675, + "score": -0.23782740036230163 + }, + "val": { + "rmse": 0.23857661998288315, + "mae": 0.13661375641822815, + "r2": 0.6198044419288635, + "score": -0.23857661998288315 + }, + "test": { + "rmse": 0.2544202590204169, + "mae": 0.17306958138942719, + "r2": 0.557673454284668, + "score": -0.2544202590204169 + } + }, + "time": "0:01:20.386711", + "diversity": { + "train": { + "mean_centered_corr": -0.01925019733607769, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04815442115068436 + }, + "val": { + "mean_centered_corr": -0.018709633499383926, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06300598382949829 + }, + "test": { + "mean_centered_corr": -0.018054556101560593, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08491437882184982 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0 + ], + "head_selection_time": "0:00:00.015002" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p00025-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p00025-evaluation/0.toml new file mode 100644 index 000000000..2f8be345d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p00025-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.00025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p00025-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p00025-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p00025-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p00025-evaluation/0/report.json new file mode 100644 index 000000000..8b8f2f72d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p00025-evaluation/0/report.json @@ -0,0 +1,203 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0p001_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23108709757774937, + "mae": 0.1223861500620842, + "r2": 0.6588343381881714, + "score": -0.23108709757774937 + }, + "val": { + "rmse": 0.23484986994593354, + "mae": 0.12684030830860138, + "r2": 0.6315895318984985, + "score": -0.23484986994593354 + }, + "test": { + "rmse": 0.23462197483665095, + "mae": 0.1482354700565338, + "r2": 0.6238362193107605, + "score": -0.23462197483665095 + } + }, + "time": "0:01:20.386711", + "diversity": { + "train": { + "mean_centered_corr": -0.01925019733607769, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04815442115068436 + }, + "val": { + "mean_centered_corr": -0.018709633499383926, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06300598382949829 + }, + "test": { + "mean_centered_corr": -0.018054556101560593, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08491437882184982 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p00025-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p00025-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p00025-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p00025-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..afa6f99ff --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p00025-greedy-heads-evaluation/0/report.json @@ -0,0 +1,217 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0p001_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2319917866834578, + "mae": 0.12356825172901154, + "r2": 0.6561578512191772, + "score": -0.2319917866834578 + }, + "val": { + "rmse": 0.23320292020753594, + "mae": 0.12550441920757294, + "r2": 0.636738657951355, + "score": -0.23320292020753594 + }, + "test": { + "rmse": 0.23031042609193297, + "mae": 0.14081889390945435, + "r2": 0.6375343799591064, + "score": -0.23031042609193297 + } + }, + "time": "0:01:20.386711", + "diversity": { + "train": { + "mean_centered_corr": -0.01925019733607769, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04815442115068436 + }, + "val": { + "mean_centered_corr": -0.018709633499383926, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06300598382949829 + }, + "test": { + "mean_centered_corr": -0.018054556101560593, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08491437882184982 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0, + 4, + 28, + 1, + 19, + 24, + 16, + 4, + 1, + 20, + 21 + ], + "head_selection_time": "0:00:00.171966" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..1e9aadbf5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0005-best-head-evaluation/0/report.json @@ -0,0 +1,207 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0p001_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2320922951078084, + "mae": 0.12688623368740082, + "r2": 0.6558598279953003, + "score": -0.2320922951078084 + }, + "val": { + "rmse": 0.2390755017179478, + "mae": 0.13819952309131622, + "r2": 0.6182127594947815, + "score": -0.2390755017179478 + }, + "test": { + "rmse": 0.25168892959044736, + "mae": 0.1720852106809616, + "r2": 0.5671197175979614, + "score": -0.25168892959044736 + } + }, + "time": "0:01:27.604837", + "diversity": { + "train": { + "mean_centered_corr": -0.018731974065303802, + "mean_pairwise_disagreement": NaN, + "member_std": 0.049796536564826965 + }, + "val": { + "mean_centered_corr": -0.019370803609490395, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06845120340585709 + }, + "test": { + "mean_centered_corr": -0.016793658956885338, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0947653204202652 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 22 + ], + "head_selection_time": "0:00:00.015528" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0005-evaluation/0.toml new file mode 100644 index 000000000..237bd5384 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0005-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.0005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0005-evaluation/0/report.json new file mode 100644 index 000000000..8110cb0cf --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0005-evaluation/0/report.json @@ -0,0 +1,203 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0p001_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2222445177940426, + "mae": 0.12026011943817139, + "r2": 0.6844443082809448, + "score": -0.2222445177940426 + }, + "val": { + "rmse": 0.23510992538054096, + "mae": 0.12966062128543854, + "r2": 0.6307731866836548, + "score": -0.23510992538054096 + }, + "test": { + "rmse": 0.24175341990615093, + "mae": 0.1596364676952362, + "r2": 0.6006213426589966, + "score": -0.24175341990615093 + } + }, + "time": "0:01:27.604837", + "diversity": { + "train": { + "mean_centered_corr": -0.018731974065303802, + "mean_pairwise_disagreement": NaN, + "member_std": 0.049796536564826965 + }, + "val": { + "mean_centered_corr": -0.019370803609490395, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06845120340585709 + }, + "test": { + "mean_centered_corr": -0.016793658956885338, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0947653204202652 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..f570919e0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0p001_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22296470056985443, + "mae": 0.12100711464881897, + "r2": 0.682395875453949, + "score": -0.22296470056985443 + }, + "val": { + "rmse": 0.2333487134502395, + "mae": 0.12775763869285583, + "r2": 0.6362842321395874, + "score": -0.2333487134502395 + }, + "test": { + "rmse": 0.2416932153437863, + "mae": 0.16084370017051697, + "r2": 0.6008202433586121, + "score": -0.2416932153437863 + } + }, + "time": "0:01:27.604837", + "diversity": { + "train": { + "mean_centered_corr": -0.018731974065303802, + "mean_pairwise_disagreement": NaN, + "member_std": 0.049796536564826965 + }, + "val": { + "mean_centered_corr": -0.019370803609490395, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06845120340585709 + }, + "test": { + "mean_centered_corr": -0.016793658956885338, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0947653204202652 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 22, + 3, + 4, + 8, + 25, + 20, + 21 + ], + "head_selection_time": "0:00:00.115215" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..8844d39fb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p001-best-head-evaluation/0/report.json @@ -0,0 +1,207 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0p001_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2255253318194067, + "mae": 0.1253618746995926, + "r2": 0.6750589609146118, + "score": -0.2255253318194067 + }, + "val": { + "rmse": 0.23949031043262045, + "mae": 0.13341857492923737, + "r2": 0.6168867349624634, + "score": -0.23949031043262045 + }, + "test": { + "rmse": 0.24155888192007524, + "mae": 0.14752615988254547, + "r2": 0.6012638807296753, + "score": -0.24155888192007524 + } + }, + "time": "0:01:30.152123", + "diversity": { + "train": { + "mean_centered_corr": -0.017563171684741974, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048281844705343246 + }, + "val": { + "mean_centered_corr": -0.01754843257367611, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07056254893541336 + }, + "test": { + "mean_centered_corr": -0.016294721513986588, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09666775166988373 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19 + ], + "head_selection_time": "0:00:00.015102" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p001-evaluation/0.toml new file mode 100644 index 000000000..d65a74bc0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p001-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.001 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p001-evaluation/0/report.json new file mode 100644 index 000000000..12a205021 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p001-evaluation/0/report.json @@ -0,0 +1,203 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0p001_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2197185345348766, + "mae": 0.11797432601451874, + "r2": 0.6915766000747681, + "score": -0.2197185345348766 + }, + "val": { + "rmse": 0.23467774706155475, + "mae": 0.1267256885766983, + "r2": 0.632129430770874, + "score": -0.23467774706155475 + }, + "test": { + "rmse": 0.23716614057842658, + "mae": 0.1508055180311203, + "r2": 0.6156339645385742, + "score": -0.23716614057842658 + } + }, + "time": "0:01:30.152123", + "diversity": { + "train": { + "mean_centered_corr": -0.017563171684741974, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048281844705343246 + }, + "val": { + "mean_centered_corr": -0.01754843257367611, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07056254893541336 + }, + "test": { + "mean_centered_corr": -0.016294721513986588, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09666775166988373 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..9d2512c60 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,219 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0p001_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.22057000931073897, + "mae": 0.11852128803730011, + "r2": 0.6891815662384033, + "score": -0.22057000931073897 + }, + "val": { + "rmse": 0.23316146280351166, + "mae": 0.12596334517002106, + "r2": 0.6368677616119385, + "score": -0.23316146280351166 + }, + "test": { + "rmse": 0.23452524296935331, + "mae": 0.14668044447898865, + "r2": 0.6241463422775269, + "score": -0.23452524296935331 + } + }, + "time": "0:01:30.152123", + "diversity": { + "train": { + "mean_centered_corr": -0.017563171684741974, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048281844705343246 + }, + "val": { + "mean_centered_corr": -0.01754843257367611, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07056254893541336 + }, + "test": { + "mean_centered_corr": -0.016294721513986588, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09666775166988373 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19, + 4, + 16, + 12, + 0, + 25, + 31, + 3, + 22, + 29, + 12, + 27, + 20 + ], + "head_selection_time": "0:00:00.202186" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0025-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0025-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0025-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0025-best-head-evaluation/0/report.json new file mode 100644 index 000000000..7e8a2da9b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0025-best-head-evaluation/0/report.json @@ -0,0 +1,207 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0p001_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2261742520907439, + "mae": 0.12224072217941284, + "r2": 0.6731863021850586, + "score": -0.2261742520907439 + }, + "val": { + "rmse": 0.2377136850270048, + "mae": 0.13327811658382416, + "r2": 0.622549831867218, + "score": -0.2377136850270048 + }, + "test": { + "rmse": 0.262345673404661, + "mae": 0.17990553379058838, + "r2": 0.5296865105628967, + "score": -0.262345673404661 + } + }, + "time": "0:01:26.505665", + "diversity": { + "train": { + "mean_centered_corr": -0.01859835349023342, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04920431599020958 + }, + "val": { + "mean_centered_corr": -0.019710257649421692, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06811030209064484 + }, + "test": { + "mean_centered_corr": -0.013359437696635723, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09920918941497803 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015205" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0025-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0025-evaluation/0.toml new file mode 100644 index 000000000..f5afb1e37 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0025-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.0025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0025-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0025-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0025-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0025-evaluation/0/report.json new file mode 100644 index 000000000..825c4e2d0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0025-evaluation/0/report.json @@ -0,0 +1,203 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0p001_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22265827445732783, + "mae": 0.11788544058799744, + "r2": 0.6832683086395264, + "score": -0.22265827445732783 + }, + "val": { + "rmse": 0.23474623366768377, + "mae": 0.1270996779203415, + "r2": 0.6319146752357483, + "score": -0.23474623366768377 + }, + "test": { + "rmse": 0.23936563514587833, + "mae": 0.15535929799079895, + "r2": 0.6084716320037842, + "score": -0.23936563514587833 + } + }, + "time": "0:01:26.505665", + "diversity": { + "train": { + "mean_centered_corr": -0.01859835349023342, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04920431599020958 + }, + "val": { + "mean_centered_corr": -0.019710257649421692, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06811030209064484 + }, + "test": { + "mean_centered_corr": -0.013359437696635723, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09920918941497803 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0025-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0025-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0025-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0025-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..325697f2f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0025-greedy-heads-evaluation/0/report.json @@ -0,0 +1,214 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0p001_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22450348389711117, + "mae": 0.11922673135995865, + "r2": 0.6779968738555908, + "score": -0.22450348389711117 + }, + "val": { + "rmse": 0.23295802298295287, + "mae": 0.1258699893951416, + "r2": 0.6375011801719666, + "score": -0.23295802298295287 + }, + "test": { + "rmse": 0.23903366020873654, + "mae": 0.15511320531368256, + "r2": 0.6095569133758545, + "score": -0.23903366020873654 + } + }, + "time": "0:01:26.505665", + "diversity": { + "train": { + "mean_centered_corr": -0.01859835349023342, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04920431599020958 + }, + "val": { + "mean_centered_corr": -0.019710257649421692, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06811030209064484 + }, + "test": { + "mean_centered_corr": -0.013359437696635723, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09920918941497803 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 22, + 12, + 16, + 21, + 19, + 25, + 29 + ], + "head_selection_time": "0:00:00.128994" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p00025-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p00025-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p00025-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p00025-best-head-evaluation/0/report.json new file mode 100644 index 000000000..d54a7312a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p00025-best-head-evaluation/0/report.json @@ -0,0 +1,207 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n1em05_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23782740036230163, + "mae": 0.13576887547969818, + "r2": 0.6386420130729675, + "score": -0.23782740036230163 + }, + "val": { + "rmse": 0.23857661998288315, + "mae": 0.13661375641822815, + "r2": 0.6198044419288635, + "score": -0.23857661998288315 + }, + "test": { + "rmse": 0.2544202590204169, + "mae": 0.17306958138942719, + "r2": 0.557673454284668, + "score": -0.2544202590204169 + } + }, + "time": "0:01:21.063765", + "diversity": { + "train": { + "mean_centered_corr": -0.01925019733607769, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04815442115068436 + }, + "val": { + "mean_centered_corr": -0.018709633499383926, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06300598382949829 + }, + "test": { + "mean_centered_corr": -0.018054556101560593, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08491437882184982 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0 + ], + "head_selection_time": "0:00:00.015192" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p00025-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p00025-evaluation/0.toml new file mode 100644 index 000000000..8bf510609 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p00025-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.00025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p00025-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p00025-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p00025-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p00025-evaluation/0/report.json new file mode 100644 index 000000000..4acc7e1b1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p00025-evaluation/0/report.json @@ -0,0 +1,203 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n1em05_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23108709757774937, + "mae": 0.1223861500620842, + "r2": 0.6588343381881714, + "score": -0.23108709757774937 + }, + "val": { + "rmse": 0.23484986994593354, + "mae": 0.12684030830860138, + "r2": 0.6315895318984985, + "score": -0.23484986994593354 + }, + "test": { + "rmse": 0.23462197483665095, + "mae": 0.1482354700565338, + "r2": 0.6238362193107605, + "score": -0.23462197483665095 + } + }, + "time": "0:01:21.063765", + "diversity": { + "train": { + "mean_centered_corr": -0.01925019733607769, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04815442115068436 + }, + "val": { + "mean_centered_corr": -0.018709633499383926, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06300598382949829 + }, + "test": { + "mean_centered_corr": -0.018054556101560593, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08491437882184982 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p00025-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p00025-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p00025-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p00025-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..2956d8bbc --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p00025-greedy-heads-evaluation/0/report.json @@ -0,0 +1,217 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n1em05_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2319917866834578, + "mae": 0.12356825172901154, + "r2": 0.6561578512191772, + "score": -0.2319917866834578 + }, + "val": { + "rmse": 0.23320292020753594, + "mae": 0.12550441920757294, + "r2": 0.636738657951355, + "score": -0.23320292020753594 + }, + "test": { + "rmse": 0.23031042609193297, + "mae": 0.14081889390945435, + "r2": 0.6375343799591064, + "score": -0.23031042609193297 + } + }, + "time": "0:01:21.063765", + "diversity": { + "train": { + "mean_centered_corr": -0.01925019733607769, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04815442115068436 + }, + "val": { + "mean_centered_corr": -0.018709633499383926, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06300598382949829 + }, + "test": { + "mean_centered_corr": -0.018054556101560593, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08491437882184982 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0, + 4, + 28, + 1, + 19, + 24, + 16, + 4, + 1, + 20, + 21 + ], + "head_selection_time": "0:00:00.173704" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..a4081fa3f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0005-best-head-evaluation/0/report.json @@ -0,0 +1,207 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n1em05_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2320922951078084, + "mae": 0.12688623368740082, + "r2": 0.6558598279953003, + "score": -0.2320922951078084 + }, + "val": { + "rmse": 0.2390755017179478, + "mae": 0.13819952309131622, + "r2": 0.6182127594947815, + "score": -0.2390755017179478 + }, + "test": { + "rmse": 0.25168892959044736, + "mae": 0.1720852106809616, + "r2": 0.5671197175979614, + "score": -0.25168892959044736 + } + }, + "time": "0:01:27.149432", + "diversity": { + "train": { + "mean_centered_corr": -0.018731974065303802, + "mean_pairwise_disagreement": NaN, + "member_std": 0.049796536564826965 + }, + "val": { + "mean_centered_corr": -0.019370803609490395, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06845120340585709 + }, + "test": { + "mean_centered_corr": -0.016793658956885338, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0947653204202652 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 22 + ], + "head_selection_time": "0:00:00.015214" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0005-evaluation/0.toml new file mode 100644 index 000000000..e8b20d26c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0005-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.0005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0005-evaluation/0/report.json new file mode 100644 index 000000000..baf1310ef --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0005-evaluation/0/report.json @@ -0,0 +1,203 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n1em05_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2222445177940426, + "mae": 0.12026011943817139, + "r2": 0.6844443082809448, + "score": -0.2222445177940426 + }, + "val": { + "rmse": 0.23510992538054096, + "mae": 0.12966062128543854, + "r2": 0.6307731866836548, + "score": -0.23510992538054096 + }, + "test": { + "rmse": 0.24175341990615093, + "mae": 0.1596364676952362, + "r2": 0.6006213426589966, + "score": -0.24175341990615093 + } + }, + "time": "0:01:27.149432", + "diversity": { + "train": { + "mean_centered_corr": -0.018731974065303802, + "mean_pairwise_disagreement": NaN, + "member_std": 0.049796536564826965 + }, + "val": { + "mean_centered_corr": -0.019370803609490395, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06845120340585709 + }, + "test": { + "mean_centered_corr": -0.016793658956885338, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0947653204202652 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..4a4d3ce2f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n1em05_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22296470056985443, + "mae": 0.12100711464881897, + "r2": 0.682395875453949, + "score": -0.22296470056985443 + }, + "val": { + "rmse": 0.2333487134502395, + "mae": 0.12775763869285583, + "r2": 0.6362842321395874, + "score": -0.2333487134502395 + }, + "test": { + "rmse": 0.2416932153437863, + "mae": 0.16084370017051697, + "r2": 0.6008202433586121, + "score": -0.2416932153437863 + } + }, + "time": "0:01:27.149432", + "diversity": { + "train": { + "mean_centered_corr": -0.018731974065303802, + "mean_pairwise_disagreement": NaN, + "member_std": 0.049796536564826965 + }, + "val": { + "mean_centered_corr": -0.019370803609490395, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06845120340585709 + }, + "test": { + "mean_centered_corr": -0.016793658956885338, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0947653204202652 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 22, + 3, + 4, + 8, + 25, + 20, + 21 + ], + "head_selection_time": "0:00:00.115778" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..67f1062dc --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p001-best-head-evaluation/0/report.json @@ -0,0 +1,207 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n1em05_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2255253318194067, + "mae": 0.1253618746995926, + "r2": 0.6750589609146118, + "score": -0.2255253318194067 + }, + "val": { + "rmse": 0.23949031043262045, + "mae": 0.13341857492923737, + "r2": 0.6168867349624634, + "score": -0.23949031043262045 + }, + "test": { + "rmse": 0.24155888192007524, + "mae": 0.14752615988254547, + "r2": 0.6012638807296753, + "score": -0.24155888192007524 + } + }, + "time": "0:01:30.041102", + "diversity": { + "train": { + "mean_centered_corr": -0.017563171684741974, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048281844705343246 + }, + "val": { + "mean_centered_corr": -0.01754843257367611, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07056254893541336 + }, + "test": { + "mean_centered_corr": -0.016294721513986588, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09666775166988373 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19 + ], + "head_selection_time": "0:00:00.015164" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p001-evaluation/0.toml new file mode 100644 index 000000000..bfd17e870 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p001-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.001 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p001-evaluation/0/report.json new file mode 100644 index 000000000..df0136fbb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p001-evaluation/0/report.json @@ -0,0 +1,203 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n1em05_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2197185345348766, + "mae": 0.11797432601451874, + "r2": 0.6915766000747681, + "score": -0.2197185345348766 + }, + "val": { + "rmse": 0.23467774706155475, + "mae": 0.1267256885766983, + "r2": 0.632129430770874, + "score": -0.23467774706155475 + }, + "test": { + "rmse": 0.23716614057842658, + "mae": 0.1508055180311203, + "r2": 0.6156339645385742, + "score": -0.23716614057842658 + } + }, + "time": "0:01:30.041102", + "diversity": { + "train": { + "mean_centered_corr": -0.017563171684741974, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048281844705343246 + }, + "val": { + "mean_centered_corr": -0.01754843257367611, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07056254893541336 + }, + "test": { + "mean_centered_corr": -0.016294721513986588, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09666775166988373 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..7d9441f30 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,219 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n1em05_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.22057000931073897, + "mae": 0.11852128803730011, + "r2": 0.6891815662384033, + "score": -0.22057000931073897 + }, + "val": { + "rmse": 0.23316146280351166, + "mae": 0.12596334517002106, + "r2": 0.6368677616119385, + "score": -0.23316146280351166 + }, + "test": { + "rmse": 0.23452524296935331, + "mae": 0.14668044447898865, + "r2": 0.6241463422775269, + "score": -0.23452524296935331 + } + }, + "time": "0:01:30.041102", + "diversity": { + "train": { + "mean_centered_corr": -0.017563171684741974, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048281844705343246 + }, + "val": { + "mean_centered_corr": -0.01754843257367611, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07056254893541336 + }, + "test": { + "mean_centered_corr": -0.016294721513986588, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09666775166988373 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19, + 4, + 16, + 12, + 0, + 25, + 31, + 3, + 22, + 29, + 12, + 27, + 20 + ], + "head_selection_time": "0:00:00.205114" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0025-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0025-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0025-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0025-best-head-evaluation/0/report.json new file mode 100644 index 000000000..9a13723e7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0025-best-head-evaluation/0/report.json @@ -0,0 +1,207 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n1em05_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2261742520907439, + "mae": 0.12224072217941284, + "r2": 0.6731863021850586, + "score": -0.2261742520907439 + }, + "val": { + "rmse": 0.2377136850270048, + "mae": 0.13327811658382416, + "r2": 0.622549831867218, + "score": -0.2377136850270048 + }, + "test": { + "rmse": 0.262345673404661, + "mae": 0.17990553379058838, + "r2": 0.5296865105628967, + "score": -0.262345673404661 + } + }, + "time": "0:01:27.660572", + "diversity": { + "train": { + "mean_centered_corr": -0.01859835349023342, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04920431599020958 + }, + "val": { + "mean_centered_corr": -0.019710257649421692, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06811030209064484 + }, + "test": { + "mean_centered_corr": -0.013359437696635723, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09920918941497803 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015449" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0025-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0025-evaluation/0.toml new file mode 100644 index 000000000..7a8746abc --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0025-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.0025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0025-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0025-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0025-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0025-evaluation/0/report.json new file mode 100644 index 000000000..f597ea970 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0025-evaluation/0/report.json @@ -0,0 +1,203 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n1em05_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22265827445732783, + "mae": 0.11788544058799744, + "r2": 0.6832683086395264, + "score": -0.22265827445732783 + }, + "val": { + "rmse": 0.23474623366768377, + "mae": 0.1270996779203415, + "r2": 0.6319146752357483, + "score": -0.23474623366768377 + }, + "test": { + "rmse": 0.23936563514587833, + "mae": 0.15535929799079895, + "r2": 0.6084716320037842, + "score": -0.23936563514587833 + } + }, + "time": "0:01:27.660572", + "diversity": { + "train": { + "mean_centered_corr": -0.01859835349023342, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04920431599020958 + }, + "val": { + "mean_centered_corr": -0.019710257649421692, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06811030209064484 + }, + "test": { + "mean_centered_corr": -0.013359437696635723, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09920918941497803 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0025-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0025-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0025-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0025-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..8b959ea4c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0025-greedy-heads-evaluation/0/report.json @@ -0,0 +1,214 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n1em05_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22450348389711117, + "mae": 0.11922673135995865, + "r2": 0.6779968738555908, + "score": -0.22450348389711117 + }, + "val": { + "rmse": 0.23295802298295287, + "mae": 0.1258699893951416, + "r2": 0.6375011801719666, + "score": -0.23295802298295287 + }, + "test": { + "rmse": 0.23903366020873654, + "mae": 0.15511320531368256, + "r2": 0.6095569133758545, + "score": -0.23903366020873654 + } + }, + "time": "0:01:27.660572", + "diversity": { + "train": { + "mean_centered_corr": -0.01859835349023342, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04920431599020958 + }, + "val": { + "mean_centered_corr": -0.019710257649421692, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06811030209064484 + }, + "test": { + "mean_centered_corr": -0.013359437696635723, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09920918941497803 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 22, + 12, + 16, + 21, + 19, + 25, + 29 + ], + "head_selection_time": "0:00:00.133344" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p00025-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p00025-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p00025-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p00025-best-head-evaluation/0/report.json new file mode 100644 index 000000000..92fd4c825 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p00025-best-head-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 962, + "metrics": { + "train": { + "rmse": 0.22433421654217983, + "mae": 0.12722031772136688, + "r2": 0.6784822940826416, + "score": -0.22433421654217983 + }, + "val": { + "rmse": 0.24006717570021247, + "mae": 0.1350351870059967, + "r2": 0.6150389313697815, + "score": -0.24006717570021247 + }, + "test": { + "rmse": 0.24244266573296758, + "mae": 0.14548884332180023, + "r2": 0.5983408689498901, + "score": -0.24244266573296758 + } + }, + "time": "0:02:07.704029", + "diversity": { + "train": { + "mean_centered_corr": -0.016253771260380745, + "mean_pairwise_disagreement": NaN, + "member_std": 0.049984753131866455 + }, + "val": { + "mean_centered_corr": -0.016636746004223824, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0729350671172142 + }, + "test": { + "mean_centered_corr": -0.01810598000884056, + "mean_pairwise_disagreement": NaN, + "member_std": 0.10638274252414703 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19 + ], + "head_selection_time": "0:00:00.015142" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p00025-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p00025-evaluation/0.toml new file mode 100644 index 000000000..f76490e2b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p00025-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.00025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p00025-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p00025-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p00025-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p00025-evaluation/0/report.json new file mode 100644 index 000000000..c8df19d94 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p00025-evaluation/0/report.json @@ -0,0 +1,209 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 962, + "metrics": { + "train": { + "rmse": 0.21573275520444352, + "mae": 0.12005273252725601, + "r2": 0.7026649713516235, + "score": -0.21573275520444352 + }, + "val": { + "rmse": 0.23499185132041592, + "mae": 0.12818463146686554, + "r2": 0.6311439871788025, + "score": -0.23499185132041592 + }, + "test": { + "rmse": 0.2394094259267782, + "mae": 0.15464657545089722, + "r2": 0.6083284020423889, + "score": -0.2394094259267782 + } + }, + "time": "0:02:07.704029", + "diversity": { + "train": { + "mean_centered_corr": -0.016253771260380745, + "mean_pairwise_disagreement": NaN, + "member_std": 0.049984753131866455 + }, + "val": { + "mean_centered_corr": -0.016636746004223824, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0729350671172142 + }, + "test": { + "mean_centered_corr": -0.01810598000884056, + "mean_pairwise_disagreement": NaN, + "member_std": 0.10638274252414703 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p00025-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p00025-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p00025-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p00025-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..2fa26438c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p00025-greedy-heads-evaluation/0/report.json @@ -0,0 +1,224 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 962, + "metrics": { + "train": { + "rmse": 0.21682843846304425, + "mae": 0.12020629644393921, + "r2": 0.6996370553970337, + "score": -0.21682843846304425 + }, + "val": { + "rmse": 0.23372927564914306, + "mae": 0.1274442821741104, + "r2": 0.6350969672203064, + "score": -0.23372927564914306 + }, + "test": { + "rmse": 0.2350238164160606, + "mae": 0.14731569588184357, + "r2": 0.6225466132164001, + "score": -0.2350238164160606 + } + }, + "time": "0:02:07.704029", + "diversity": { + "train": { + "mean_centered_corr": -0.016253771260380745, + "mean_pairwise_disagreement": NaN, + "member_std": 0.049984753131866455 + }, + "val": { + "mean_centered_corr": -0.016636746004223824, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0729350671172142 + }, + "test": { + "mean_centered_corr": -0.01810598000884056, + "mean_pairwise_disagreement": NaN, + "member_std": 0.10638274252414703 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19, + 16, + 20, + 21, + 4, + 12, + 27, + 28, + 2, + 22, + 31, + 8 + ], + "head_selection_time": "0:00:00.189880" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..bdf7ec1d9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0005-best-head-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.23104360784438088, + "mae": 0.12508618831634521, + "r2": 0.6589627265930176, + "score": -0.23104360784438088 + }, + "val": { + "rmse": 0.2383891939899224, + "mae": 0.13487277925014496, + "r2": 0.6204016208648682, + "score": -0.2383891939899224 + }, + "test": { + "rmse": 0.24401096715766643, + "mae": 0.15994176268577576, + "r2": 0.5931275486946106, + "score": -0.24401096715766643 + } + }, + "time": "0:01:59.937521", + "diversity": { + "train": { + "mean_centered_corr": -0.01880013383924961, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04820944741368294 + }, + "val": { + "mean_centered_corr": -0.01936255395412445, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06788419187068939 + }, + "test": { + "mean_centered_corr": -0.017522644251585007, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09551511704921722 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 22 + ], + "head_selection_time": "0:00:00.015282" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0005-evaluation/0.toml new file mode 100644 index 000000000..6312a247d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0005-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.0005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0005-evaluation/0/report.json new file mode 100644 index 000000000..30c1cc9a8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0005-evaluation/0/report.json @@ -0,0 +1,209 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22253845679462023, + "mae": 0.11854175478219986, + "r2": 0.6836090683937073, + "score": -0.22253845679462023 + }, + "val": { + "rmse": 0.2346688971022881, + "mae": 0.12585745751857758, + "r2": 0.6321571469306946, + "score": -0.2346688971022881 + }, + "test": { + "rmse": 0.23375485551605374, + "mae": 0.14507132768630981, + "r2": 0.626611590385437, + "score": -0.23375485551605374 + } + }, + "time": "0:01:59.937521", + "diversity": { + "train": { + "mean_centered_corr": -0.01880013383924961, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04820944741368294 + }, + "val": { + "mean_centered_corr": -0.01936255395412445, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06788419187068939 + }, + "test": { + "mean_centered_corr": -0.017522644251585007, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09551511704921722 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..1ec0b4622 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,221 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22311397819116993, + "mae": 0.12033727020025253, + "r2": 0.681970477104187, + "score": -0.22311397819116993 + }, + "val": { + "rmse": 0.23289370528688721, + "mae": 0.1270957887172699, + "r2": 0.6377012729644775, + "score": -0.23289370528688721 + }, + "test": { + "rmse": 0.2404508003490849, + "mae": 0.15944166481494904, + "r2": 0.6049136519432068, + "score": -0.2404508003490849 + } + }, + "time": "0:01:59.937521", + "diversity": { + "train": { + "mean_centered_corr": -0.01880013383924961, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04820944741368294 + }, + "val": { + "mean_centered_corr": -0.01936255395412445, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06788419187068939 + }, + "test": { + "mean_centered_corr": -0.017522644251585007, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09551511704921722 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 22, + 4, + 25, + 28, + 12, + 1, + 18, + 4, + 16 + ], + "head_selection_time": "0:00:00.144082" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..ac1e0d186 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p001-best-head-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22728253913691077, + "mae": 0.12302348017692566, + "r2": 0.6699756383895874, + "score": -0.22728253913691077 + }, + "val": { + "rmse": 0.23897535038319878, + "mae": 0.13313278555870056, + "r2": 0.6185325384140015, + "score": -0.23897535038319878 + }, + "test": { + "rmse": 0.24383994851478993, + "mae": 0.14601385593414307, + "r2": 0.5936976671218872, + "score": -0.24383994851478993 + } + }, + "time": "0:01:59.114238", + "diversity": { + "train": { + "mean_centered_corr": -0.018884949386119843, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047628145664930344 + }, + "val": { + "mean_centered_corr": -0.017485102638602257, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06956880539655685 + }, + "test": { + "mean_centered_corr": -0.01598832942545414, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09292414039373398 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19 + ], + "head_selection_time": "0:00:00.015354" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p001-evaluation/0.toml new file mode 100644 index 000000000..812605f91 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p001-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.001 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p001-evaluation/0/report.json new file mode 100644 index 000000000..46a42870c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p001-evaluation/0/report.json @@ -0,0 +1,209 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2233263689720867, + "mae": 0.11833805590867996, + "r2": 0.6813646554946899, + "score": -0.2233263689720867 + }, + "val": { + "rmse": 0.23475735787864294, + "mae": 0.12769214808940887, + "r2": 0.6318798065185547, + "score": -0.23475735787864294 + }, + "test": { + "rmse": 0.23961932837948463, + "mae": 0.15490485727787018, + "r2": 0.607641339302063, + "score": -0.23961932837948463 + } + }, + "time": "0:01:59.114238", + "diversity": { + "train": { + "mean_centered_corr": -0.018884949386119843, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047628145664930344 + }, + "val": { + "mean_centered_corr": -0.017485102638602257, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06956880539655685 + }, + "test": { + "mean_centered_corr": -0.01598832942545414, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09292414039373398 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..ed5eb58b1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,224 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22469448467375502, + "mae": 0.12008367478847504, + "r2": 0.6774487495422363, + "score": -0.22469448467375502 + }, + "val": { + "rmse": 0.23343394819363691, + "mae": 0.12686417996883392, + "r2": 0.6360185146331787, + "score": -0.23343394819363691 + }, + "test": { + "rmse": 0.2365000863679966, + "mae": 0.15199944376945496, + "r2": 0.6177898645401001, + "score": -0.2365000863679966 + } + }, + "time": "0:01:59.114238", + "diversity": { + "train": { + "mean_centered_corr": -0.018884949386119843, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047628145664930344 + }, + "val": { + "mean_centered_corr": -0.017485102638602257, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06956880539655685 + }, + "test": { + "mean_centered_corr": -0.01598832942545414, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09292414039373398 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19, + 21, + 4, + 12, + 16, + 28, + 20, + 29, + 18, + 12, + 16, + 21 + ], + "head_selection_time": "0:00:00.192700" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0025-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0025-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0025-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0025-best-head-evaluation/0/report.json new file mode 100644 index 000000000..7beab8ba3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0025-best-head-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2255459539475908, + "mae": 0.12257809191942215, + "r2": 0.6749995350837708, + "score": -0.2255459539475908 + }, + "val": { + "rmse": 0.2401098998840954, + "mae": 0.13103342056274414, + "r2": 0.6149019002914429, + "score": -0.2401098998840954 + }, + "test": { + "rmse": 0.24179294965081416, + "mae": 0.14307323098182678, + "r2": 0.6004906892776489, + "score": -0.24179294965081416 + } + }, + "time": "0:02:05.400277", + "diversity": { + "train": { + "mean_centered_corr": -0.01803121715784073, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04816228523850441 + }, + "val": { + "mean_centered_corr": -0.01891929842531681, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06890527904033661 + }, + "test": { + "mean_centered_corr": -0.017366157844662666, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0932842418551445 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19 + ], + "head_selection_time": "0:00:00.015342" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0025-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0025-evaluation/0.toml new file mode 100644 index 000000000..bfe3860e5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0025-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.0025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0025-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0025-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0025-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0025-evaluation/0/report.json new file mode 100644 index 000000000..c0c72f7b4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0025-evaluation/0/report.json @@ -0,0 +1,209 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.21942591125322533, + "mae": 0.1180998906493187, + "r2": 0.6923975944519043, + "score": -0.21942591125322533 + }, + "val": { + "rmse": 0.23456885744734607, + "mae": 0.12691855430603027, + "r2": 0.6324707269668579, + "score": -0.23456885744734607 + }, + "test": { + "rmse": 0.23654787235061092, + "mae": 0.15111033618450165, + "r2": 0.6176353693008423, + "score": -0.23654787235061092 + } + }, + "time": "0:02:05.400277", + "diversity": { + "train": { + "mean_centered_corr": -0.01803121715784073, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04816228523850441 + }, + "val": { + "mean_centered_corr": -0.01891929842531681, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06890527904033661 + }, + "test": { + "mean_centered_corr": -0.017366157844662666, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0932842418551445 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0025-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0025-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0025-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0025-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..2e87bc24c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0025-greedy-heads-evaluation/0/report.json @@ -0,0 +1,224 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.22062443706127244, + "mae": 0.1181936264038086, + "r2": 0.6890281438827515, + "score": -0.22062443706127244 + }, + "val": { + "rmse": 0.23332136469753356, + "mae": 0.126436248421669, + "r2": 0.636369526386261, + "score": -0.23332136469753356 + }, + "test": { + "rmse": 0.2364625390906921, + "mae": 0.15152248740196228, + "r2": 0.6179112195968628, + "score": -0.2364625390906921 + } + }, + "time": "0:02:05.400277", + "diversity": { + "train": { + "mean_centered_corr": -0.01803121715784073, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04816228523850441 + }, + "val": { + "mean_centered_corr": -0.01891929842531681, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06890527904033661 + }, + "test": { + "mean_centered_corr": -0.017366157844662666, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0932842418551445 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19, + 4, + 21, + 25, + 12, + 16, + 28, + 31, + 22, + 20, + 0, + 25 + ], + "head_selection_time": "0:00:00.189725" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p00025-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p00025-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p00025-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p00025-best-head-evaluation/0/report.json new file mode 100644 index 000000000..975a359bd --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p00025-best-head-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0p0001_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.23099349784472964, + "mae": 0.13101767003536224, + "r2": 0.6591106653213501, + "score": -0.23099349784472964 + }, + "val": { + "rmse": 0.23704860354665752, + "mae": 0.13406512141227722, + "r2": 0.6246589422225952, + "score": -0.23704860354665752 + }, + "test": { + "rmse": 0.23874426241915847, + "mae": 0.15374092757701874, + "r2": 0.610501766204834, + "score": -0.23874426241915847 + } + }, + "time": "0:01:59.843910", + "diversity": { + "train": { + "mean_centered_corr": -0.018132619559764862, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04782529175281525 + }, + "val": { + "mean_centered_corr": -0.018767302855849266, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06245988979935646 + }, + "test": { + "mean_centered_corr": -0.016851365566253662, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08885195106267929 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 16 + ], + "head_selection_time": "0:00:00.016150" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p00025-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p00025-evaluation/0.toml new file mode 100644 index 000000000..5d5322fb9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p00025-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.00025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p00025-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p00025-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p00025-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p00025-evaluation/0/report.json new file mode 100644 index 000000000..0f642a60c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p00025-evaluation/0/report.json @@ -0,0 +1,209 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0p0001_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22186950085769538, + "mae": 0.11825541406869888, + "r2": 0.6855083703994751, + "score": -0.22186950085769538 + }, + "val": { + "rmse": 0.23496530422551093, + "mae": 0.12728473544120789, + "r2": 0.6312273144721985, + "score": -0.23496530422551093 + }, + "test": { + "rmse": 0.23600846732737993, + "mae": 0.14936019480228424, + "r2": 0.6193772554397583, + "score": -0.23600846732737993 + } + }, + "time": "0:01:59.843910", + "diversity": { + "train": { + "mean_centered_corr": -0.018132619559764862, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04782529175281525 + }, + "val": { + "mean_centered_corr": -0.018767302855849266, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06245988979935646 + }, + "test": { + "mean_centered_corr": -0.016851365566253662, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08885195106267929 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p00025-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p00025-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p00025-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p00025-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..81bbd4812 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p00025-greedy-heads-evaluation/0/report.json @@ -0,0 +1,221 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0p0001_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22194279588607752, + "mae": 0.11977781355381012, + "r2": 0.6853005290031433, + "score": -0.22194279588607752 + }, + "val": { + "rmse": 0.23324589552748487, + "mae": 0.12741298973560333, + "r2": 0.6366047263145447, + "score": -0.23324589552748487 + }, + "test": { + "rmse": 0.23661615616090015, + "mae": 0.1510157734155655, + "r2": 0.6174145936965942, + "score": -0.23661615616090015 + } + }, + "time": "0:01:59.843910", + "diversity": { + "train": { + "mean_centered_corr": -0.018132619559764862, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04782529175281525 + }, + "val": { + "mean_centered_corr": -0.018767302855849266, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06245988979935646 + }, + "test": { + "mean_centered_corr": -0.016851365566253662, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08885195106267929 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 16, + 4, + 19, + 22, + 12, + 25, + 30, + 28, + 4 + ], + "head_selection_time": "0:00:00.154365" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..e26e031b4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0005-best-head-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0p0001_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2313707126836422, + "mae": 0.1336175799369812, + "r2": 0.657996416091919, + "score": -0.2313707126836422 + }, + "val": { + "rmse": 0.23835052202476453, + "mae": 0.13680434226989746, + "r2": 0.6205247640609741, + "score": -0.23835052202476453 + }, + "test": { + "rmse": 0.23699550345285816, + "mae": 0.1488882154226303, + "r2": 0.6161868572235107, + "score": -0.23699550345285816 + } + }, + "time": "0:02:00.384580", + "diversity": { + "train": { + "mean_centered_corr": -0.018761252984404564, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04712963104248047 + }, + "val": { + "mean_centered_corr": -0.017407886683940887, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07094229757785797 + }, + "test": { + "mean_centered_corr": -0.01455279067158699, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09494736045598984 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 16 + ], + "head_selection_time": "0:00:00.015364" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0005-evaluation/0.toml new file mode 100644 index 000000000..fa39890f9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0005-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.0005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0005-evaluation/0/report.json new file mode 100644 index 000000000..1d93fe8c8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0005-evaluation/0/report.json @@ -0,0 +1,209 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0p0001_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.222181256913413, + "mae": 0.11818218231201172, + "r2": 0.6846239566802979, + "score": -0.222181256913413 + }, + "val": { + "rmse": 0.23480699780789907, + "mae": 0.12649822235107422, + "r2": 0.6317241191864014, + "score": -0.23480699780789907 + }, + "test": { + "rmse": 0.23801010369895165, + "mae": 0.153095081448555, + "r2": 0.6128935813903809, + "score": -0.23801010369895165 + } + }, + "time": "0:02:00.384580", + "diversity": { + "train": { + "mean_centered_corr": -0.018761252984404564, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04712963104248047 + }, + "val": { + "mean_centered_corr": -0.017407886683940887, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07094229757785797 + }, + "test": { + "mean_centered_corr": -0.01455279067158699, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09494736045598984 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..d124019f8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,220 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0p0001_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2223846971387422, + "mae": 0.11872012168169022, + "r2": 0.6840461492538452, + "score": -0.2223846971387422 + }, + "val": { + "rmse": 0.2333328522027255, + "mae": 0.12700752913951874, + "r2": 0.636333703994751, + "score": -0.2333328522027255 + }, + "test": { + "rmse": 0.238922935773872, + "mae": 0.1549610048532486, + "r2": 0.6099185943603516, + "score": -0.238922935773872 + } + }, + "time": "0:02:00.384580", + "diversity": { + "train": { + "mean_centered_corr": -0.018761252984404564, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04712963104248047 + }, + "val": { + "mean_centered_corr": -0.017407886683940887, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07094229757785797 + }, + "test": { + "mean_centered_corr": -0.01455279067158699, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09494736045598984 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 16, + 19, + 25, + 12, + 4, + 18, + 31, + 29 + ], + "head_selection_time": "0:00:00.130977" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..ed6677fc6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p001-best-head-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0p0001_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24213441143837863, + "mae": 0.12409187108278275, + "r2": 0.6254352331161499, + "score": -0.24213441143837863 + }, + "val": { + "rmse": 0.23848516306571665, + "mae": 0.13054722547531128, + "r2": 0.6200958490371704, + "score": -0.23848516306571665 + }, + "test": { + "rmse": 0.24223068067706838, + "mae": 0.15242092311382294, + "r2": 0.5990428924560547, + "score": -0.24223068067706838 + } + }, + "time": "0:01:52.170887", + "diversity": { + "train": { + "mean_centered_corr": -0.018726564943790436, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04575931653380394 + }, + "val": { + "mean_centered_corr": -0.019934404641389847, + "mean_pairwise_disagreement": NaN, + "member_std": 0.061195019632577896 + }, + "test": { + "mean_centered_corr": -0.018395286053419113, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08133252710103989 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015563" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p001-evaluation/0.toml new file mode 100644 index 000000000..a2e0d9d10 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p001-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.001 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p001-evaluation/0/report.json new file mode 100644 index 000000000..72762527c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p001-evaluation/0/report.json @@ -0,0 +1,209 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0p0001_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2329486599171296, + "mae": 0.11878164857625961, + "r2": 0.653315544128418, + "score": -0.2329486599171296 + }, + "val": { + "rmse": 0.2356879335438401, + "mae": 0.12487198412418365, + "r2": 0.6289555430412292, + "score": -0.2356879335438401 + }, + "test": { + "rmse": 0.23192085632684412, + "mae": 0.1407395601272583, + "r2": 0.6324476599693298, + "score": -0.23192085632684412 + } + }, + "time": "0:01:52.170887", + "diversity": { + "train": { + "mean_centered_corr": -0.018726564943790436, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04575931653380394 + }, + "val": { + "mean_centered_corr": -0.019934404641389847, + "mean_pairwise_disagreement": NaN, + "member_std": 0.061195019632577896 + }, + "test": { + "mean_centered_corr": -0.018395286053419113, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08133252710103989 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..c8381a924 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,223 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0p0001_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23254022069636512, + "mae": 0.12052810937166214, + "r2": 0.6545302271842957, + "score": -0.23254022069636512 + }, + "val": { + "rmse": 0.23318228828211082, + "mae": 0.12569577991962433, + "r2": 0.6368029117584229, + "score": -0.23318228828211082 + }, + "test": { + "rmse": 0.23425661912073115, + "mae": 0.14765594899654388, + "r2": 0.6250068545341492, + "score": -0.23425661912073115 + } + }, + "time": "0:01:52.170887", + "diversity": { + "train": { + "mean_centered_corr": -0.018726564943790436, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04575931653380394 + }, + "val": { + "mean_centered_corr": -0.019934404641389847, + "mean_pairwise_disagreement": NaN, + "member_std": 0.061195019632577896 + }, + "test": { + "mean_centered_corr": -0.018395286053419113, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08133252710103989 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 0, + 20, + 12, + 21, + 25, + 19, + 20, + 0, + 24, + 16 + ], + "head_selection_time": "0:00:00.177504" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0025-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0025-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0025-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0025-best-head-evaluation/0/report.json new file mode 100644 index 000000000..c53576bc5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0025-best-head-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0p0001_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2385315752740614, + "mae": 0.13105398416519165, + "r2": 0.6364989876747131, + "score": -0.2385315752740614 + }, + "val": { + "rmse": 0.23800908632875115, + "mae": 0.13678862154483795, + "r2": 0.6216111183166504, + "score": -0.23800908632875115 + }, + "test": { + "rmse": 0.2395995521584212, + "mae": 0.15617705881595612, + "r2": 0.6077060699462891, + "score": -0.2395995521584212 + } + }, + "time": "0:01:51.161126", + "diversity": { + "train": { + "mean_centered_corr": -0.01903150975704193, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04610341414809227 + }, + "val": { + "mean_centered_corr": -0.0190847497433424, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06286956369876862 + }, + "test": { + "mean_centered_corr": -0.01705675758421421, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08540277928113937 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 16 + ], + "head_selection_time": "0:00:00.015566" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0025-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0025-evaluation/0.toml new file mode 100644 index 000000000..fd91e5498 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0025-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.0025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0025-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0025-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0025-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0025-evaluation/0/report.json new file mode 100644 index 000000000..7232e98aa --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0025-evaluation/0/report.json @@ -0,0 +1,209 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0p0001_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23162405942251374, + "mae": 0.11943608522415161, + "r2": 0.6572470664978027, + "score": -0.23162405942251374 + }, + "val": { + "rmse": 0.2352761982909868, + "mae": 0.1248830109834671, + "r2": 0.6302508115768433, + "score": -0.2352761982909868 + }, + "test": { + "rmse": 0.2333683170052332, + "mae": 0.14440852403640747, + "r2": 0.6278454065322876, + "score": -0.2333683170052332 + } + }, + "time": "0:01:51.161126", + "diversity": { + "train": { + "mean_centered_corr": -0.01903150975704193, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04610341414809227 + }, + "val": { + "mean_centered_corr": -0.0190847497433424, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06286956369876862 + }, + "test": { + "mean_centered_corr": -0.01705675758421421, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08540277928113937 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0025-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0025-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0025-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0025-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..21677580e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0025-greedy-heads-evaluation/0/report.json @@ -0,0 +1,220 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0p0001_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23244876064945358, + "mae": 0.12024175375699997, + "r2": 0.6548019647598267, + "score": -0.23244876064945358 + }, + "val": { + "rmse": 0.2331158511435021, + "mae": 0.12569448351860046, + "r2": 0.637009859085083, + "score": -0.2331158511435021 + }, + "test": { + "rmse": 0.23340534056347034, + "mae": 0.14711007475852966, + "r2": 0.6277272701263428, + "score": -0.23340534056347034 + } + }, + "time": "0:01:51.161126", + "diversity": { + "train": { + "mean_centered_corr": -0.01903150975704193, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04610341414809227 + }, + "val": { + "mean_centered_corr": -0.0190847497433424, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06286956369876862 + }, + "test": { + "mean_centered_corr": -0.01705675758421421, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08540277928113937 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 16, + 28, + 19, + 4, + 18, + 1, + 25, + 20 + ], + "head_selection_time": "0:00:00.131487" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p00025-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p00025-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p00025-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p00025-best-head-evaluation/0/report.json new file mode 100644 index 000000000..b148197ca --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p00025-best-head-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0p001_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23600330571914405, + "mae": 0.13268455862998962, + "r2": 0.6441638469696045, + "score": -0.23600330571914405 + }, + "val": { + "rmse": 0.23803283687571922, + "mae": 0.1352454274892807, + "r2": 0.6215356588363647, + "score": -0.23803283687571922 + }, + "test": { + "rmse": 0.273329277420384, + "mae": 0.1970231831073761, + "r2": 0.48948097229003906, + "score": -0.273329277420384 + } + }, + "time": "0:01:51.858305", + "diversity": { + "train": { + "mean_centered_corr": -0.01871478743851185, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046988409012556076 + }, + "val": { + "mean_centered_corr": -0.019513292238116264, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06265565007925034 + }, + "test": { + "mean_centered_corr": -0.018462633714079857, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08088342845439911 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0 + ], + "head_selection_time": "0:00:00.015035" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p00025-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p00025-evaluation/0.toml new file mode 100644 index 000000000..a2a06e681 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p00025-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.00025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p00025-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p00025-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p00025-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p00025-evaluation/0/report.json new file mode 100644 index 000000000..ca5cac233 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p00025-evaluation/0/report.json @@ -0,0 +1,209 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0p001_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23085188135419868, + "mae": 0.12120585143566132, + "r2": 0.6595284938812256, + "score": -0.23085188135419868 + }, + "val": { + "rmse": 0.23478980709452008, + "mae": 0.12477234750986099, + "r2": 0.6317780017852783, + "score": -0.23478980709452008 + }, + "test": { + "rmse": 0.2332225120575295, + "mae": 0.1445438116788864, + "r2": 0.6283103227615356, + "score": -0.2332225120575295 + } + }, + "time": "0:01:51.858305", + "diversity": { + "train": { + "mean_centered_corr": -0.01871478743851185, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046988409012556076 + }, + "val": { + "mean_centered_corr": -0.019513292238116264, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06265565007925034 + }, + "test": { + "mean_centered_corr": -0.018462633714079857, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08088342845439911 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p00025-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p00025-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p00025-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p00025-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..f4dda662e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p00025-greedy-heads-evaluation/0/report.json @@ -0,0 +1,220 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0p001_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23141464802856693, + "mae": 0.12309345602989197, + "r2": 0.6578664779663086, + "score": -0.23141464802856693 + }, + "val": { + "rmse": 0.23289638454580067, + "mae": 0.12518544495105743, + "r2": 0.6376929879188538, + "score": -0.23289638454580067 + }, + "test": { + "rmse": 0.23281509730791192, + "mae": 0.1455666571855545, + "r2": 0.6296077966690063, + "score": -0.23281509730791192 + } + }, + "time": "0:01:51.858305", + "diversity": { + "train": { + "mean_centered_corr": -0.01871478743851185, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046988409012556076 + }, + "val": { + "mean_centered_corr": -0.019513292238116264, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06265565007925034 + }, + "test": { + "mean_centered_corr": -0.018462633714079857, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08088342845439911 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0, + 19, + 4, + 16, + 12, + 18, + 1, + 20 + ], + "head_selection_time": "0:00:00.129381" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..96de473ea --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0005-best-head-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0p001_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22534824415842816, + "mae": 0.12385565787553787, + "r2": 0.6755690574645996, + "score": -0.22534824415842816 + }, + "val": { + "rmse": 0.24035605783658937, + "mae": 0.13962619006633759, + "r2": 0.6141119003295898, + "score": -0.24035605783658937 + }, + "test": { + "rmse": 0.2779356858657707, + "mae": 0.19685637950897217, + "r2": 0.4721285104751587, + "score": -0.2779356858657707 + } + }, + "time": "0:02:00.459838", + "diversity": { + "train": { + "mean_centered_corr": -0.018568821251392365, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047292761504650116 + }, + "val": { + "mean_centered_corr": -0.01956503838300705, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07274748384952545 + }, + "test": { + "mean_centered_corr": -0.01600496470928192, + "mean_pairwise_disagreement": NaN, + "member_std": 0.1011786088347435 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015299" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0005-evaluation/0.toml new file mode 100644 index 000000000..e261269fd --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0005-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.0005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0005-evaluation/0/report.json new file mode 100644 index 000000000..ab224775e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0005-evaluation/0/report.json @@ -0,0 +1,209 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0p001_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22215539246899518, + "mae": 0.11718897521495819, + "r2": 0.6846973896026611, + "score": -0.22215539246899518 + }, + "val": { + "rmse": 0.23527326904017234, + "mae": 0.12683525681495667, + "r2": 0.6302599906921387, + "score": -0.23527326904017234 + }, + "test": { + "rmse": 0.24074289728147005, + "mae": 0.15624843537807465, + "r2": 0.6039531230926514, + "score": -0.24074289728147005 + } + }, + "time": "0:02:00.459838", + "diversity": { + "train": { + "mean_centered_corr": -0.018568821251392365, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047292761504650116 + }, + "val": { + "mean_centered_corr": -0.01956503838300705, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07274748384952545 + }, + "test": { + "mean_centered_corr": -0.01600496470928192, + "mean_pairwise_disagreement": NaN, + "member_std": 0.1011786088347435 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..6ae6be381 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,218 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0p001_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22264168507638651, + "mae": 0.11877916008234024, + "r2": 0.6833154559135437, + "score": -0.22264168507638651 + }, + "val": { + "rmse": 0.23367927924707155, + "mae": 0.12780483067035675, + "r2": 0.6352530717849731, + "score": -0.23367927924707155 + }, + "test": { + "rmse": 0.24524323007832846, + "mae": 0.16379818320274353, + "r2": 0.5890077352523804, + "score": -0.24524323007832846 + } + }, + "time": "0:02:00.459838", + "diversity": { + "train": { + "mean_centered_corr": -0.018568821251392365, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047292761504650116 + }, + "val": { + "mean_centered_corr": -0.01956503838300705, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07274748384952545 + }, + "test": { + "mean_centered_corr": -0.01600496470928192, + "mean_pairwise_disagreement": NaN, + "member_std": 0.1011786088347435 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 19, + 1, + 12, + 29, + 25 + ], + "head_selection_time": "0:00:00.101612" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..174b506e1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p001-best-head-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0p001_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22760798675130503, + "mae": 0.12891143560409546, + "r2": 0.6690298318862915, + "score": -0.22760798675130503 + }, + "val": { + "rmse": 0.23789169956459427, + "mae": 0.13463006913661957, + "r2": 0.6219843029975891, + "score": -0.23789169956459427 + }, + "test": { + "rmse": 0.23950414627424274, + "mae": 0.14432719349861145, + "r2": 0.6080183982849121, + "score": -0.23950414627424274 + } + }, + "time": "0:02:00.788746", + "diversity": { + "train": { + "mean_centered_corr": -0.01867086812853813, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04923015832901001 + }, + "val": { + "mean_centered_corr": -0.019777491688728333, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06746035814285278 + }, + "test": { + "mean_centered_corr": -0.014284917153418064, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09343798458576202 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19 + ], + "head_selection_time": "0:00:00.015396" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p001-evaluation/0.toml new file mode 100644 index 000000000..49d9094ee --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p001-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.001 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p001-evaluation/0/report.json new file mode 100644 index 000000000..ea35c38ac --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p001-evaluation/0/report.json @@ -0,0 +1,209 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0p001_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22229246068030295, + "mae": 0.12095138430595398, + "r2": 0.6843081712722778, + "score": -0.22229246068030295 + }, + "val": { + "rmse": 0.23507535721736414, + "mae": 0.1287076324224472, + "r2": 0.6308817863464355, + "score": -0.23507535721736414 + }, + "test": { + "rmse": 0.2433693260742304, + "mae": 0.1609274446964264, + "r2": 0.5952645540237427, + "score": -0.2433693260742304 + } + }, + "time": "0:02:00.788746", + "diversity": { + "train": { + "mean_centered_corr": -0.01867086812853813, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04923015832901001 + }, + "val": { + "mean_centered_corr": -0.019777491688728333, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06746035814285278 + }, + "test": { + "mean_centered_corr": -0.014284917153418064, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09343798458576202 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..76e167010 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,221 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0p001_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2227845825987346, + "mae": 0.12195254862308502, + "r2": 0.682908833026886, + "score": -0.2227845825987346 + }, + "val": { + "rmse": 0.23367845823808855, + "mae": 0.1281619518995285, + "r2": 0.6352556347846985, + "score": -0.23367845823808855 + }, + "test": { + "rmse": 0.24371457881808845, + "mae": 0.1601243019104004, + "r2": 0.5941153764724731, + "score": -0.24371457881808845 + } + }, + "time": "0:02:00.788746", + "diversity": { + "train": { + "mean_centered_corr": -0.01867086812853813, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04923015832901001 + }, + "val": { + "mean_centered_corr": -0.019777491688728333, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06746035814285278 + }, + "test": { + "mean_centered_corr": -0.014284917153418064, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09343798458576202 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19, + 12, + 28, + 25, + 8, + 4, + 12, + 16, + 17 + ], + "head_selection_time": "0:00:00.145937" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0025-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0025-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0025-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0025-best-head-evaluation/0/report.json new file mode 100644 index 000000000..69abc5cba --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0025-best-head-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0p001_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23691756457376573, + "mae": 0.130058154463768, + "r2": 0.6414015293121338, + "score": -0.23691756457376573 + }, + "val": { + "rmse": 0.23784205343302714, + "mae": 0.13620039820671082, + "r2": 0.6221420764923096, + "score": -0.23784205343302714 + }, + "test": { + "rmse": 0.23492100218498793, + "mae": 0.14837363362312317, + "r2": 0.622876763343811, + "score": -0.23492100218498793 + } + }, + "time": "0:01:51.789439", + "diversity": { + "train": { + "mean_centered_corr": -0.018733546137809753, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04638450965285301 + }, + "val": { + "mean_centered_corr": -0.019742805510759354, + "mean_pairwise_disagreement": NaN, + "member_std": 0.060568567365407944 + }, + "test": { + "mean_centered_corr": -0.018700512126088142, + "mean_pairwise_disagreement": NaN, + "member_std": 0.078513003885746 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 16 + ], + "head_selection_time": "0:00:00.015130" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0025-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0025-evaluation/0.toml new file mode 100644 index 000000000..180a4eb3d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0025-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.0025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0025-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0025-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0025-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0025-evaluation/0/report.json new file mode 100644 index 000000000..cae422be4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0025-evaluation/0/report.json @@ -0,0 +1,209 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0p001_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23100547201881677, + "mae": 0.12135384976863861, + "r2": 0.6590753197669983, + "score": -0.23100547201881677 + }, + "val": { + "rmse": 0.23446964895916758, + "mae": 0.1270645558834076, + "r2": 0.6327815055847168, + "score": -0.23446964895916758 + }, + "test": { + "rmse": 0.23505021426230255, + "mae": 0.1497301310300827, + "r2": 0.6224617958068848, + "score": -0.23505021426230255 + } + }, + "time": "0:01:51.789439", + "diversity": { + "train": { + "mean_centered_corr": -0.018733546137809753, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04638450965285301 + }, + "val": { + "mean_centered_corr": -0.019742805510759354, + "mean_pairwise_disagreement": NaN, + "member_std": 0.060568567365407944 + }, + "test": { + "mean_centered_corr": -0.018700512126088142, + "mean_pairwise_disagreement": NaN, + "member_std": 0.078513003885746 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0025-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0025-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0025-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0025-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..a39bb4840 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0025-greedy-heads-evaluation/0/report.json @@ -0,0 +1,223 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0p001_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23208180560739772, + "mae": 0.12161051481962204, + "r2": 0.655890941619873, + "score": -0.23208180560739772 + }, + "val": { + "rmse": 0.23278002032353212, + "mae": 0.1258585900068283, + "r2": 0.6380549669265747, + "score": -0.23278002032353212 + }, + "test": { + "rmse": 0.23227273339288673, + "mae": 0.14668183028697968, + "r2": 0.6313314437866211, + "score": -0.23227273339288673 + } + }, + "time": "0:01:51.789439", + "diversity": { + "train": { + "mean_centered_corr": -0.018733546137809753, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04638450965285301 + }, + "val": { + "mean_centered_corr": -0.019742805510759354, + "mean_pairwise_disagreement": NaN, + "member_std": 0.060568567365407944 + }, + "test": { + "mean_centered_corr": -0.018700512126088142, + "mean_pairwise_disagreement": NaN, + "member_std": 0.078513003885746 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 16, + 28, + 4, + 12, + 20, + 21, + 0, + 24, + 4, + 1, + 19 + ], + "head_selection_time": "0:00:00.172089" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p00025-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p00025-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p00025-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p00025-best-head-evaluation/0/report.json new file mode 100644 index 000000000..024b13351 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p00025-best-head-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n1em05_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23738592821530582, + "mae": 0.12199530750513077, + "r2": 0.6399823427200317, + "score": -0.23738592821530582 + }, + "val": { + "rmse": 0.23736952064793038, + "mae": 0.12993131577968597, + "r2": 0.6236419677734375, + "score": -0.23736952064793038 + }, + "test": { + "rmse": 0.24275107338262392, + "mae": 0.15340644121170044, + "r2": 0.5973182916641235, + "score": -0.24275107338262392 + } + }, + "time": "0:01:52.147419", + "diversity": { + "train": { + "mean_centered_corr": -0.018815485760569572, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046015676110982895 + }, + "val": { + "mean_centered_corr": -0.018479591235518456, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06330302357673645 + }, + "test": { + "mean_centered_corr": -0.01912987418472767, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08374166488647461 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015222" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p00025-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p00025-evaluation/0.toml new file mode 100644 index 000000000..423740a5b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p00025-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.00025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p00025-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p00025-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p00025-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p00025-evaluation/0/report.json new file mode 100644 index 000000000..12a1c544e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p00025-evaluation/0/report.json @@ -0,0 +1,209 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n1em05_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23060420468503218, + "mae": 0.11958978325128555, + "r2": 0.6602587103843689, + "score": -0.23060420468503218 + }, + "val": { + "rmse": 0.23516112255992166, + "mae": 0.12509571015834808, + "r2": 0.6306123733520508, + "score": -0.23516112255992166 + }, + "test": { + "rmse": 0.23211789488202353, + "mae": 0.14202691614627838, + "r2": 0.6318228244781494, + "score": -0.23211789488202353 + } + }, + "time": "0:01:52.147419", + "diversity": { + "train": { + "mean_centered_corr": -0.018815485760569572, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046015676110982895 + }, + "val": { + "mean_centered_corr": -0.018479591235518456, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06330302357673645 + }, + "test": { + "mean_centered_corr": -0.01912987418472767, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08374166488647461 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p00025-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p00025-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p00025-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p00025-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..44baedc4e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p00025-greedy-heads-evaluation/0/report.json @@ -0,0 +1,219 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n1em05_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23160982522236812, + "mae": 0.12151593714952469, + "r2": 0.6572891473770142, + "score": -0.23160982522236812 + }, + "val": { + "rmse": 0.23322604208950684, + "mae": 0.12618425488471985, + "r2": 0.6366665959358215, + "score": -0.23322604208950684 + }, + "test": { + "rmse": 0.23242508060584816, + "mae": 0.1440695971250534, + "r2": 0.630847692489624, + "score": -0.23242508060584816 + } + }, + "time": "0:01:52.147419", + "diversity": { + "train": { + "mean_centered_corr": -0.018815485760569572, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046015676110982895 + }, + "val": { + "mean_centered_corr": -0.018479591235518456, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06330302357673645 + }, + "test": { + "mean_centered_corr": -0.01912987418472767, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08374166488647461 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 16, + 19, + 0, + 21, + 16, + 12 + ], + "head_selection_time": "0:00:00.114905" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..c7de4a402 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0005-best-head-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n1em05_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22784794156293847, + "mae": 0.12659133970737457, + "r2": 0.6683316230773926, + "score": -0.22784794156293847 + }, + "val": { + "rmse": 0.2393372540058719, + "mae": 0.1392020285129547, + "r2": 0.6173763275146484, + "score": -0.2393372540058719 + }, + "test": { + "rmse": 0.2906284957593463, + "mae": 0.21522651612758636, + "r2": 0.4228137731552124, + "score": -0.2906284957593463 + } + }, + "time": "0:01:59.475961", + "diversity": { + "train": { + "mean_centered_corr": -0.019331008195877075, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04783523082733154 + }, + "val": { + "mean_centered_corr": -0.019431548193097115, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06572002172470093 + }, + "test": { + "mean_centered_corr": -0.016879823058843613, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08989977091550827 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25 + ], + "head_selection_time": "0:00:00.015390" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0005-evaluation/0.toml new file mode 100644 index 000000000..8c97c9fd2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0005-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.0005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0005-evaluation/0/report.json new file mode 100644 index 000000000..a9d12d5ce --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0005-evaluation/0/report.json @@ -0,0 +1,209 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n1em05_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22281919339573966, + "mae": 0.11718059331178665, + "r2": 0.6828103065490723, + "score": -0.22281919339573966 + }, + "val": { + "rmse": 0.23464263099787872, + "mae": 0.12664183974266052, + "r2": 0.6322394609451294, + "score": -0.23464263099787872 + }, + "test": { + "rmse": 0.23482863711969398, + "mae": 0.14775922894477844, + "r2": 0.6231732368469238, + "score": -0.23482863711969398 + } + }, + "time": "0:01:59.475961", + "diversity": { + "train": { + "mean_centered_corr": -0.019331008195877075, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04783523082733154 + }, + "val": { + "mean_centered_corr": -0.019431548193097115, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06572002172470093 + }, + "test": { + "mean_centered_corr": -0.016879823058843613, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08989977091550827 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..e9bacf5cd --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,225 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n1em05_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22351819228584932, + "mae": 0.11853836476802826, + "r2": 0.6808170676231384, + "score": -0.22351819228584932 + }, + "val": { + "rmse": 0.23272791516423366, + "mae": 0.1259242445230484, + "r2": 0.6382169723510742, + "score": -0.23272791516423366 + }, + "test": { + "rmse": 0.23258942887652473, + "mae": 0.14525124430656433, + "r2": 0.630325436592102, + "score": -0.23258942887652473 + } + }, + "time": "0:01:59.475961", + "diversity": { + "train": { + "mean_centered_corr": -0.019331008195877075, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04783523082733154 + }, + "val": { + "mean_centered_corr": -0.019431548193097115, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06572002172470093 + }, + "test": { + "mean_centered_corr": -0.016879823058843613, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08989977091550827 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25, + 12, + 16, + 19, + 4, + 29, + 12, + 28, + 20, + 21, + 19, + 16, + 12 + ], + "head_selection_time": "0:00:00.204657" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..5173871f0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p001-best-head-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n1em05_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23885602106177764, + "mae": 0.12304521352052689, + "r2": 0.6355094313621521, + "score": -0.23885602106177764 + }, + "val": { + "rmse": 0.23826622524732466, + "mae": 0.13159264624118805, + "r2": 0.6207931041717529, + "score": -0.23826622524732466 + }, + "test": { + "rmse": 0.2354789879616611, + "mae": 0.14298199117183685, + "r2": 0.62108314037323, + "score": -0.2354789879616611 + } + }, + "time": "0:01:51.731542", + "diversity": { + "train": { + "mean_centered_corr": -0.01938508450984955, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04722513630986214 + }, + "val": { + "mean_centered_corr": -0.01980144903063774, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06155725568532944 + }, + "test": { + "mean_centered_corr": -0.01934674195945263, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08168060332536697 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015102" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p001-evaluation/0.toml new file mode 100644 index 000000000..dd37e163a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p001-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.001 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p001-evaluation/0/report.json new file mode 100644 index 000000000..1067761ef --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p001-evaluation/0/report.json @@ -0,0 +1,209 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n1em05_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23113198136255483, + "mae": 0.1195804625749588, + "r2": 0.6587017774581909, + "score": -0.23113198136255483 + }, + "val": { + "rmse": 0.23461772747586743, + "mae": 0.12575800716876984, + "r2": 0.6323175430297852, + "score": -0.23461772747586743 + }, + "test": { + "rmse": 0.23292056846137055, + "mae": 0.14504367113113403, + "r2": 0.6292721033096313, + "score": -0.23292056846137055 + } + }, + "time": "0:01:51.731542", + "diversity": { + "train": { + "mean_centered_corr": -0.01938508450984955, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04722513630986214 + }, + "val": { + "mean_centered_corr": -0.01980144903063774, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06155725568532944 + }, + "test": { + "mean_centered_corr": -0.01934674195945263, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08168060332536697 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..55de1b3f6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,223 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n1em05_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23143950990251744, + "mae": 0.11987357586622238, + "r2": 0.6577929854393005, + "score": -0.23143950990251744 + }, + "val": { + "rmse": 0.23311639447730767, + "mae": 0.12623491883277893, + "r2": 0.6370081305503845, + "score": -0.23311639447730767 + }, + "test": { + "rmse": 0.2316950082866651, + "mae": 0.1445348560810089, + "r2": 0.6331631541252136, + "score": -0.2316950082866651 + } + }, + "time": "0:01:51.731542", + "diversity": { + "train": { + "mean_centered_corr": -0.01938508450984955, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04722513630986214 + }, + "val": { + "mean_centered_corr": -0.01980144903063774, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06155725568532944 + }, + "test": { + "mean_centered_corr": -0.01934674195945263, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08168060332536697 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 19, + 18, + 20, + 0, + 28, + 1, + 21, + 12, + 16, + 20 + ], + "head_selection_time": "0:00:00.172745" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0025-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0025-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0025-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0025-best-head-evaluation/0/report.json new file mode 100644 index 000000000..f5cbf59f1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0025-best-head-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n1em05_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24032697986190826, + "mae": 0.12809138000011444, + "r2": 0.6310063004493713, + "score": -0.24032697986190826 + }, + "val": { + "rmse": 0.23826509952531, + "mae": 0.13267768919467926, + "r2": 0.6207966804504395, + "score": -0.23826509952531 + }, + "test": { + "rmse": 0.23646242093377673, + "mae": 0.14509999752044678, + "r2": 0.6179115772247314, + "score": -0.23646242093377673 + } + }, + "time": "0:01:52.868935", + "diversity": { + "train": { + "mean_centered_corr": -0.018483418971300125, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04600879177451134 + }, + "val": { + "mean_centered_corr": -0.019203318282961845, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06106279790401459 + }, + "test": { + "mean_centered_corr": -0.016553888097405434, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07980619370937347 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015166" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0025-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0025-evaluation/0.toml new file mode 100644 index 000000000..e12a2c399 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0025-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.0025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0025-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0025-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0025-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0025-evaluation/0/report.json new file mode 100644 index 000000000..a8928f6bd --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0025-evaluation/0/report.json @@ -0,0 +1,209 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n1em05_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23176596757874662, + "mae": 0.12220603227615356, + "r2": 0.656826913356781, + "score": -0.23176596757874662 + }, + "val": { + "rmse": 0.23469026342966967, + "mae": 0.126294806599617, + "r2": 0.6320902109146118, + "score": -0.23469026342966967 + }, + "test": { + "rmse": 0.23333669987162928, + "mae": 0.1461082249879837, + "r2": 0.6279462575912476, + "score": -0.23333669987162928 + } + }, + "time": "0:01:52.868935", + "diversity": { + "train": { + "mean_centered_corr": -0.018483418971300125, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04600879177451134 + }, + "val": { + "mean_centered_corr": -0.019203318282961845, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06106279790401459 + }, + "test": { + "mean_centered_corr": -0.016553888097405434, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07980619370937347 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0025-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0025-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0025-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0025-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..646d15121 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0025-greedy-heads-evaluation/0/report.json @@ -0,0 +1,219 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n1em05_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23300054783459956, + "mae": 0.12348049134016037, + "r2": 0.6531611680984497, + "score": -0.23300054783459956 + }, + "val": { + "rmse": 0.23303048406462032, + "mae": 0.12674884498119354, + "r2": 0.6372756361961365, + "score": -0.23303048406462032 + }, + "test": { + "rmse": 0.23273007610764737, + "mae": 0.14747364819049835, + "r2": 0.6298782229423523, + "score": -0.23273007610764737 + } + }, + "time": "0:01:52.868935", + "diversity": { + "train": { + "mean_centered_corr": -0.018483418971300125, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04600879177451134 + }, + "val": { + "mean_centered_corr": -0.019203318282961845, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06106279790401459 + }, + "test": { + "mean_centered_corr": -0.016553888097405434, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07980619370937347 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 20, + 0, + 28, + 1, + 12, + 16 + ], + "head_selection_time": "0:00:00.114933" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p00025-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p00025-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p00025-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p00025-best-head-evaluation/0/report.json new file mode 100644 index 000000000..350fafda8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p00025-best-head-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2369720026477417, + "mae": 0.12161877006292343, + "r2": 0.6412367224693298, + "score": -0.2369720026477417 + }, + "val": { + "rmse": 0.23818537080734506, + "mae": 0.13046246767044067, + "r2": 0.6210504174232483, + "score": -0.23818537080734506 + }, + "test": { + "rmse": 0.23894844295090592, + "mae": 0.1457148939371109, + "r2": 0.6098352670669556, + "score": -0.23894844295090592 + } + }, + "time": "0:02:41.714436", + "diversity": { + "train": { + "mean_centered_corr": -0.01929367519915104, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047684069722890854 + }, + "val": { + "mean_centered_corr": -0.01931084133684635, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06489446014165878 + }, + "test": { + "mean_centered_corr": -0.016153957694768906, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0877329483628273 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015071" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p00025-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p00025-evaluation/0.toml new file mode 100644 index 000000000..9720c0f9b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p00025-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.00025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p00025-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p00025-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p00025-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p00025-evaluation/0/report.json new file mode 100644 index 000000000..396b8b8e0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p00025-evaluation/0/report.json @@ -0,0 +1,209 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23113544661823018, + "mae": 0.12141359597444534, + "r2": 0.6586915850639343, + "score": -0.23113544661823018 + }, + "val": { + "rmse": 0.23541561220714985, + "mae": 0.12720422446727753, + "r2": 0.629812479019165, + "score": -0.23541561220714985 + }, + "test": { + "rmse": 0.23578183426225294, + "mae": 0.14981894195079803, + "r2": 0.620107889175415, + "score": -0.23578183426225294 + } + }, + "time": "0:02:41.714436", + "diversity": { + "train": { + "mean_centered_corr": -0.01929367519915104, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047684069722890854 + }, + "val": { + "mean_centered_corr": -0.01931084133684635, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06489446014165878 + }, + "test": { + "mean_centered_corr": -0.016153957694768906, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0877329483628273 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p00025-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p00025-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p00025-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p00025-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..d0c204b9b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p00025-greedy-heads-evaluation/0/report.json @@ -0,0 +1,220 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23155468155329886, + "mae": 0.12149085104465485, + "r2": 0.6574523448944092, + "score": -0.23155468155329886 + }, + "val": { + "rmse": 0.2337721462391934, + "mae": 0.1271917074918747, + "r2": 0.6349630951881409, + "score": -0.2337721462391934 + }, + "test": { + "rmse": 0.2337674053617211, + "mae": 0.1486269235610962, + "r2": 0.6265714168548584, + "score": -0.2337674053617211 + } + }, + "time": "0:02:41.714436", + "diversity": { + "train": { + "mean_centered_corr": -0.01929367519915104, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047684069722890854 + }, + "val": { + "mean_centered_corr": -0.01931084133684635, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06489446014165878 + }, + "test": { + "mean_centered_corr": -0.016153957694768906, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0877329483628273 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 0, + 28, + 20, + 18, + 1, + 29, + 16 + ], + "head_selection_time": "0:00:00.128427" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..6e119ee90 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0005-best-head-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2384340312375474, + "mae": 0.12352686375379562, + "r2": 0.636796236038208, + "score": -0.2384340312375474 + }, + "val": { + "rmse": 0.23826995416351013, + "mae": 0.13112401962280273, + "r2": 0.6207812428474426, + "score": -0.23826995416351013 + }, + "test": { + "rmse": 0.23722919786139868, + "mae": 0.13967889547348022, + "r2": 0.6154295802116394, + "score": -0.23722919786139868 + } + }, + "time": "0:02:42.229099", + "diversity": { + "train": { + "mean_centered_corr": -0.019007209688425064, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04895399883389473 + }, + "val": { + "mean_centered_corr": -0.01975170709192753, + "mean_pairwise_disagreement": NaN, + "member_std": 0.065635085105896 + }, + "test": { + "mean_centered_corr": -0.01898389309644699, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08565043658018112 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015367" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0005-evaluation/0.toml new file mode 100644 index 000000000..70afd9cf8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0005-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.0005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0005-evaluation/0/report.json new file mode 100644 index 000000000..00d884281 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0005-evaluation/0/report.json @@ -0,0 +1,209 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2313063804097323, + "mae": 0.123744435608387, + "r2": 0.6581865549087524, + "score": -0.2313063804097323 + }, + "val": { + "rmse": 0.23501861733133902, + "mae": 0.12678450345993042, + "r2": 0.631060004234314, + "score": -0.23501861733133902 + }, + "test": { + "rmse": 0.23373303710107815, + "mae": 0.14622008800506592, + "r2": 0.6266812086105347, + "score": -0.23373303710107815 + } + }, + "time": "0:02:42.229099", + "diversity": { + "train": { + "mean_centered_corr": -0.019007209688425064, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04895399883389473 + }, + "val": { + "mean_centered_corr": -0.01975170709192753, + "mean_pairwise_disagreement": NaN, + "member_std": 0.065635085105896 + }, + "test": { + "mean_centered_corr": -0.01898389309644699, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08565043658018112 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..69116c88f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,218 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23205218044116352, + "mae": 0.12285098433494568, + "r2": 0.655978798866272, + "score": -0.23205218044116352 + }, + "val": { + "rmse": 0.23341469329024547, + "mae": 0.12756626307964325, + "r2": 0.6360785961151123, + "score": -0.23341469329024547 + }, + "test": { + "rmse": 0.23271509315390915, + "mae": 0.14598432183265686, + "r2": 0.6299258470535278, + "score": -0.23271509315390915 + } + }, + "time": "0:02:42.229099", + "diversity": { + "train": { + "mean_centered_corr": -0.019007209688425064, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04895399883389473 + }, + "val": { + "mean_centered_corr": -0.01975170709192753, + "mean_pairwise_disagreement": NaN, + "member_std": 0.065635085105896 + }, + "test": { + "mean_centered_corr": -0.01898389309644699, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08565043658018112 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 1, + 18, + 0, + 20, + 21 + ], + "head_selection_time": "0:00:00.102363" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..11e10d745 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p001-best-head-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22814835272103823, + "mae": 0.12647606432437897, + "r2": 0.6674563884735107, + "score": -0.22814835272103823 + }, + "val": { + "rmse": 0.23860257792551076, + "mae": 0.1318465769290924, + "r2": 0.6197217106819153, + "score": -0.23860257792551076 + }, + "test": { + "rmse": 0.30043019206533295, + "mae": 0.22363102436065674, + "r2": 0.3832250237464905, + "score": -0.30043019206533295 + } + }, + "time": "0:02:55.073822", + "diversity": { + "train": { + "mean_centered_corr": -0.018881838768720627, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04876598343253136 + }, + "val": { + "mean_centered_corr": -0.01626105234026909, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07298418134450912 + }, + "test": { + "mean_centered_corr": -0.016811326146125793, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0990188717842102 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25 + ], + "head_selection_time": "0:00:00.015371" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p001-evaluation/0.toml new file mode 100644 index 000000000..b75922aea --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p001-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.001 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p001-evaluation/0/report.json new file mode 100644 index 000000000..7c20cf7f3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p001-evaluation/0/report.json @@ -0,0 +1,209 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22307667441316298, + "mae": 0.11820665746927261, + "r2": 0.6820768117904663, + "score": -0.22307667441316298 + }, + "val": { + "rmse": 0.23496360777291245, + "mae": 0.12653271853923798, + "r2": 0.6312326192855835, + "score": -0.23496360777291245 + }, + "test": { + "rmse": 0.23914605469445158, + "mae": 0.15438953042030334, + "r2": 0.6091896891593933, + "score": -0.23914605469445158 + } + }, + "time": "0:02:55.073822", + "diversity": { + "train": { + "mean_centered_corr": -0.018881838768720627, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04876598343253136 + }, + "val": { + "mean_centered_corr": -0.01626105234026909, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07298418134450912 + }, + "test": { + "mean_centered_corr": -0.016811326146125793, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0990188717842102 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..50acbd003 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,219 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2248165916254, + "mae": 0.1193905919790268, + "r2": 0.6770980954170227, + "score": -0.2248165916254 + }, + "val": { + "rmse": 0.2336154232368663, + "mae": 0.124774269759655, + "r2": 0.635452389717102, + "score": -0.2336154232368663 + }, + "test": { + "rmse": 0.24064628828924506, + "mae": 0.1569826304912567, + "r2": 0.6042709350585938, + "score": -0.24064628828924506 + } + }, + "time": "0:02:55.073822", + "diversity": { + "train": { + "mean_centered_corr": -0.018881838768720627, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04876598343253136 + }, + "val": { + "mean_centered_corr": -0.01626105234026909, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07298418134450912 + }, + "test": { + "mean_centered_corr": -0.016811326146125793, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0990188717842102 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25, + 22, + 12, + 4, + 15, + 16, + 8 + ], + "head_selection_time": "0:00:00.116993" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0025-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0025-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0025-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0025-best-head-evaluation/0/report.json new file mode 100644 index 000000000..89dfbd742 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0025-best-head-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2322114103746433, + "mae": 0.1252126395702362, + "r2": 0.6555064916610718, + "score": -0.2322114103746433 + }, + "val": { + "rmse": 0.23785759049732064, + "mae": 0.13416989147663116, + "r2": 0.6220927238464355, + "score": -0.23785759049732064 + }, + "test": { + "rmse": 0.23589133259357026, + "mae": 0.1475115269422531, + "r2": 0.6197549104690552, + "score": -0.23589133259357026 + } + }, + "time": "0:02:41.937734", + "diversity": { + "train": { + "mean_centered_corr": -0.019183777272701263, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048386890441179276 + }, + "val": { + "mean_centered_corr": -0.018359092995524406, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06268003582954407 + }, + "test": { + "mean_centered_corr": -0.017718803137540817, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08186227828264236 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 20 + ], + "head_selection_time": "0:00:00.015353" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0025-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0025-evaluation/0.toml new file mode 100644 index 000000000..77c7a7bbb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0025-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.0025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0025-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0025-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0025-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0025-evaluation/0/report.json new file mode 100644 index 000000000..0b0f41ad0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0025-evaluation/0/report.json @@ -0,0 +1,209 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23080944483078117, + "mae": 0.12102995067834854, + "r2": 0.6596536636352539, + "score": -0.23080944483078117 + }, + "val": { + "rmse": 0.23463987641889242, + "mae": 0.1267199069261551, + "r2": 0.6322481036186218, + "score": -0.23463987641889242 + }, + "test": { + "rmse": 0.23470931052134267, + "mae": 0.14773716032505035, + "r2": 0.6235561370849609, + "score": -0.23470931052134267 + } + }, + "time": "0:02:41.937734", + "diversity": { + "train": { + "mean_centered_corr": -0.019183777272701263, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048386890441179276 + }, + "val": { + "mean_centered_corr": -0.018359092995524406, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06268003582954407 + }, + "test": { + "mean_centered_corr": -0.017718803137540817, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08186227828264236 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0025-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0025-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0025-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0025-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..21032cc0a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0025-greedy-heads-evaluation/0/report.json @@ -0,0 +1,220 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23172323223879757, + "mae": 0.12236517667770386, + "r2": 0.6569534540176392, + "score": -0.23172323223879757 + }, + "val": { + "rmse": 0.23269480219963232, + "mae": 0.1264098584651947, + "r2": 0.6383199095726013, + "score": -0.23269480219963232 + }, + "test": { + "rmse": 0.23239933834285403, + "mae": 0.1455182433128357, + "r2": 0.6309294700622559, + "score": -0.23239933834285403 + } + }, + "time": "0:02:41.937734", + "diversity": { + "train": { + "mean_centered_corr": -0.019183777272701263, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048386890441179276 + }, + "val": { + "mean_centered_corr": -0.018359092995524406, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06268003582954407 + }, + "test": { + "mean_centered_corr": -0.017718803137540817, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08186227828264236 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 20, + 1, + 4, + 28, + 19, + 21, + 16, + 0 + ], + "head_selection_time": "0:00:00.131351" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p00025-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p00025-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p00025-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p00025-best-head-evaluation/0/report.json new file mode 100644 index 000000000..8759b300a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p00025-best-head-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0p0001_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22972785536803275, + "mae": 0.12533853948116302, + "r2": 0.662835955619812, + "score": -0.22972785536803275 + }, + "val": { + "rmse": 0.23957945554469914, + "mae": 0.13747090101242065, + "r2": 0.6166014671325684, + "score": -0.23957945554469914 + }, + "test": { + "rmse": 0.2496147390992142, + "mae": 0.1673688292503357, + "r2": 0.574225127696991, + "score": -0.2496147390992142 + } + }, + "time": "0:02:54.258816", + "diversity": { + "train": { + "mean_centered_corr": -0.018549814820289612, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047382600605487823 + }, + "val": { + "mean_centered_corr": -0.019721152260899544, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06767292320728302 + }, + "test": { + "mean_centered_corr": -0.01792355626821518, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0957256406545639 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 22 + ], + "head_selection_time": "0:00:00.015165" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p00025-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p00025-evaluation/0.toml new file mode 100644 index 000000000..f46785b6a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p00025-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.00025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p00025-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p00025-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p00025-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p00025-evaluation/0/report.json new file mode 100644 index 000000000..74c149180 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p00025-evaluation/0/report.json @@ -0,0 +1,209 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0p0001_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22091751319825467, + "mae": 0.11859732866287231, + "r2": 0.688201367855072, + "score": -0.22091751319825467 + }, + "val": { + "rmse": 0.23513454704634784, + "mae": 0.12659163773059845, + "r2": 0.6306958794593811, + "score": -0.23513454704634784 + }, + "test": { + "rmse": 0.2392125690267148, + "mae": 0.15437257289886475, + "r2": 0.6089722514152527, + "score": -0.2392125690267148 + } + }, + "time": "0:02:54.258816", + "diversity": { + "train": { + "mean_centered_corr": -0.018549814820289612, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047382600605487823 + }, + "val": { + "mean_centered_corr": -0.019721152260899544, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06767292320728302 + }, + "test": { + "mean_centered_corr": -0.01792355626821518, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0957256406545639 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p00025-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p00025-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p00025-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p00025-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..6e296ff89 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p00025-greedy-heads-evaluation/0/report.json @@ -0,0 +1,220 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0p0001_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22177797398882887, + "mae": 0.12039296329021454, + "r2": 0.6857677698135376, + "score": -0.22177797398882887 + }, + "val": { + "rmse": 0.23353144315489308, + "mae": 0.1272912323474884, + "r2": 0.6357144117355347, + "score": -0.23353144315489308 + }, + "test": { + "rmse": 0.2478765142940872, + "mae": 0.16908910870552063, + "r2": 0.5801343321800232, + "score": -0.2478765142940872 + } + }, + "time": "0:02:54.258816", + "diversity": { + "train": { + "mean_centered_corr": -0.018549814820289612, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047382600605487823 + }, + "val": { + "mean_centered_corr": -0.019721152260899544, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06767292320728302 + }, + "test": { + "mean_centered_corr": -0.01792355626821518, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0957256406545639 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 22, + 12, + 16, + 27, + 25, + 4, + 20, + 18 + ], + "head_selection_time": "0:00:00.131334" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..2aa5df0cb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0005-best-head-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0p0001_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23493290303475395, + "mae": 0.13081301748752594, + "r2": 0.6473843455314636, + "score": -0.23493290303475395 + }, + "val": { + "rmse": 0.2382764815807058, + "mae": 0.12931407988071442, + "r2": 0.620760440826416, + "score": -0.2382764815807058 + }, + "test": { + "rmse": 0.24121307650092602, + "mae": 0.14123845100402832, + "r2": 0.6024046540260315, + "score": -0.24121307650092602 + } + }, + "time": "0:02:41.290051", + "diversity": { + "train": { + "mean_centered_corr": -0.01915917731821537, + "mean_pairwise_disagreement": NaN, + "member_std": 0.044233281165361404 + }, + "val": { + "mean_centered_corr": -0.01909327693283558, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06557415425777435 + }, + "test": { + "mean_centered_corr": -0.01792578585445881, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08976518362760544 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19 + ], + "head_selection_time": "0:00:00.018212" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0005-evaluation/0.toml new file mode 100644 index 000000000..1892f3234 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0005-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.0005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0005-evaluation/0/report.json new file mode 100644 index 000000000..632edb3d9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0005-evaluation/0/report.json @@ -0,0 +1,209 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0p0001_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23079886474485123, + "mae": 0.11978797614574432, + "r2": 0.6596848964691162, + "score": -0.23079886474485123 + }, + "val": { + "rmse": 0.2354849520464804, + "mae": 0.12464786320924759, + "r2": 0.6295943260192871, + "score": -0.2354849520464804 + }, + "test": { + "rmse": 0.23141729611827572, + "mae": 0.13986103236675262, + "r2": 0.6340420246124268, + "score": -0.23141729611827572 + } + }, + "time": "0:02:41.290051", + "diversity": { + "train": { + "mean_centered_corr": -0.01915917731821537, + "mean_pairwise_disagreement": NaN, + "member_std": 0.044233281165361404 + }, + "val": { + "mean_centered_corr": -0.01909327693283558, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06557415425777435 + }, + "test": { + "mean_centered_corr": -0.01792578585445881, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08976518362760544 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..92e257186 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,223 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0p0001_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2315166218673027, + "mae": 0.11995172500610352, + "r2": 0.6575649380683899, + "score": -0.2315166218673027 + }, + "val": { + "rmse": 0.23269733966625106, + "mae": 0.12517677247524261, + "r2": 0.6383119821548462, + "score": -0.23269733966625106 + }, + "test": { + "rmse": 0.23483414974881817, + "mae": 0.14970079064369202, + "r2": 0.6231555938720703, + "score": -0.23483414974881817 + } + }, + "time": "0:02:41.290051", + "diversity": { + "train": { + "mean_centered_corr": -0.01915917731821537, + "mean_pairwise_disagreement": NaN, + "member_std": 0.044233281165361404 + }, + "val": { + "mean_centered_corr": -0.01909327693283558, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06557415425777435 + }, + "test": { + "mean_centered_corr": -0.01792578585445881, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08976518362760544 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19, + 18, + 12, + 28, + 4, + 25, + 31, + 18, + 21, + 4, + 12 + ], + "head_selection_time": "0:00:00.182946" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..4766259c2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p001-best-head-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0p0001_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22476767062044256, + "mae": 0.1237136721611023, + "r2": 0.6772385835647583, + "score": -0.22476767062044256 + }, + "val": { + "rmse": 0.2387734473283714, + "mae": 0.13085761666297913, + "r2": 0.6191768646240234, + "score": -0.2387734473283714 + }, + "test": { + "rmse": 0.23954579674332094, + "mae": 0.14230574667453766, + "r2": 0.6078820824623108, + "score": -0.23954579674332094 + } + }, + "time": "0:02:55.252644", + "diversity": { + "train": { + "mean_centered_corr": -0.018560394644737244, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04619770124554634 + }, + "val": { + "mean_centered_corr": -0.01867174729704857, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06765128672122955 + }, + "test": { + "mean_centered_corr": -0.01507267914712429, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09221170097589493 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19 + ], + "head_selection_time": "0:00:00.015386" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p001-evaluation/0.toml new file mode 100644 index 000000000..7097b4f27 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p001-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.001 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p001-evaluation/0/report.json new file mode 100644 index 000000000..054e40dad --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p001-evaluation/0/report.json @@ -0,0 +1,209 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0p0001_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22135730441992338, + "mae": 0.11825704574584961, + "r2": 0.6869587302207947, + "score": -0.22135730441992338 + }, + "val": { + "rmse": 0.23554663270650014, + "mae": 0.1282879263162613, + "r2": 0.6294003129005432, + "score": -0.23554663270650014 + }, + "test": { + "rmse": 0.24084128499653581, + "mae": 0.15699797868728638, + "r2": 0.6036293506622314, + "score": -0.24084128499653581 + } + }, + "time": "0:02:55.252644", + "diversity": { + "train": { + "mean_centered_corr": -0.018560394644737244, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04619770124554634 + }, + "val": { + "mean_centered_corr": -0.01867174729704857, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06765128672122955 + }, + "test": { + "mean_centered_corr": -0.01507267914712429, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09221170097589493 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..e953f4f28 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,225 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0p0001_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2225935831054593, + "mae": 0.11814232915639877, + "r2": 0.6834522485733032, + "score": -0.2225935831054593 + }, + "val": { + "rmse": 0.23413401771618858, + "mae": 0.12737752497196198, + "r2": 0.6338320970535278, + "score": -0.23413401771618858 + }, + "test": { + "rmse": 0.23885206734563352, + "mae": 0.15390992164611816, + "r2": 0.6101499795913696, + "score": -0.23885206734563352 + } + }, + "time": "0:02:55.252644", + "diversity": { + "train": { + "mean_centered_corr": -0.018560394644737244, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04619770124554634 + }, + "val": { + "mean_centered_corr": -0.01867174729704857, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06765128672122955 + }, + "test": { + "mean_centered_corr": -0.01507267914712429, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09221170097589493 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19, + 22, + 18, + 4, + 27, + 8, + 17, + 12, + 16, + 29, + 20, + 21, + 22 + ], + "head_selection_time": "0:00:00.204447" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0025-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0025-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0025-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0025-best-head-evaluation/0/report.json new file mode 100644 index 000000000..c2d6fb60b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0025-best-head-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0p0001_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 740, + "metrics": { + "train": { + "rmse": 0.23212312683947872, + "mae": 0.12872183322906494, + "r2": 0.6557683944702148, + "score": -0.23212312683947872 + }, + "val": { + "rmse": 0.2393572230996531, + "mae": 0.13352128863334656, + "r2": 0.6173124313354492, + "score": -0.2393572230996531 + }, + "test": { + "rmse": 0.23868415725996342, + "mae": 0.1462382674217224, + "r2": 0.610697865486145, + "score": -0.23868415725996342 + } + }, + "time": "0:02:49.580435", + "diversity": { + "train": { + "mean_centered_corr": -0.01846710592508316, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04601765796542168 + }, + "val": { + "mean_centered_corr": -0.018632514402270317, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06924604624509811 + }, + "test": { + "mean_centered_corr": -0.017972121015191078, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09207665920257568 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19 + ], + "head_selection_time": "0:00:00.015228" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0025-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0025-evaluation/0.toml new file mode 100644 index 000000000..13ab81f6d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0025-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.0025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0025-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0025-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0025-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0025-evaluation/0/report.json new file mode 100644 index 000000000..3fd44c48f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0025-evaluation/0/report.json @@ -0,0 +1,209 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0p0001_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 740, + "metrics": { + "train": { + "rmse": 0.22632894316351912, + "mae": 0.1234244555234909, + "r2": 0.6727391481399536, + "score": -0.22632894316351912 + }, + "val": { + "rmse": 0.23546283513255054, + "mae": 0.12725704908370972, + "r2": 0.6296639442443848, + "score": -0.23546283513255054 + }, + "test": { + "rmse": 0.23448563984262677, + "mae": 0.14775648713111877, + "r2": 0.6242733001708984, + "score": -0.23448563984262677 + } + }, + "time": "0:02:49.580435", + "diversity": { + "train": { + "mean_centered_corr": -0.01846710592508316, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04601765796542168 + }, + "val": { + "mean_centered_corr": -0.018632514402270317, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06924604624509811 + }, + "test": { + "mean_centered_corr": -0.017972121015191078, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09207665920257568 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0025-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0025-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0025-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0025-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..23ff3622f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0025-greedy-heads-evaluation/0/report.json @@ -0,0 +1,221 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0p0001_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 740, + "metrics": { + "train": { + "rmse": 0.22564756699293684, + "mae": 0.12296506017446518, + "r2": 0.6747066378593445, + "score": -0.22564756699293684 + }, + "val": { + "rmse": 0.23411584672739014, + "mae": 0.12701718509197235, + "r2": 0.6338889598846436, + "score": -0.23411584672739014 + }, + "test": { + "rmse": 0.23627903325948496, + "mae": 0.15204240381717682, + "r2": 0.6185040473937988, + "score": -0.23627903325948496 + } + }, + "time": "0:02:49.580435", + "diversity": { + "train": { + "mean_centered_corr": -0.01846710592508316, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04601765796542168 + }, + "val": { + "mean_centered_corr": -0.018632514402270317, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06924604624509811 + }, + "test": { + "mean_centered_corr": -0.017972121015191078, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09207665920257568 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19, + 28, + 12, + 20, + 18, + 25, + 4, + 5, + 12 + ], + "head_selection_time": "0:00:00.145152" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p00025-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p00025-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p00025-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p00025-best-head-evaluation/0/report.json new file mode 100644 index 000000000..627b71fc0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p00025-best-head-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0p001_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22625887190827382, + "mae": 0.12710008025169373, + "r2": 0.6729417443275452, + "score": -0.22625887190827382 + }, + "val": { + "rmse": 0.23889975711739106, + "mae": 0.13583312928676605, + "r2": 0.6187738180160522, + "score": -0.23889975711739106 + }, + "test": { + "rmse": 0.23704205803317221, + "mae": 0.14239075779914856, + "r2": 0.616036057472229, + "score": -0.23704205803317221 + } + }, + "time": "0:02:52.457107", + "diversity": { + "train": { + "mean_centered_corr": -0.01820620521903038, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04663686826825142 + }, + "val": { + "mean_centered_corr": -0.01946132257580757, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06672783941030502 + }, + "test": { + "mean_centered_corr": -0.017525583505630493, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09117421507835388 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19 + ], + "head_selection_time": "0:00:00.015231" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p00025-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p00025-evaluation/0.toml new file mode 100644 index 000000000..3bc8abdac --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p00025-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.00025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p00025-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p00025-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p00025-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p00025-evaluation/0/report.json new file mode 100644 index 000000000..0b69654e8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p00025-evaluation/0/report.json @@ -0,0 +1,209 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0p001_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22173162515690617, + "mae": 0.11826841533184052, + "r2": 0.6858991384506226, + "score": -0.22173162515690617 + }, + "val": { + "rmse": 0.234735553294385, + "mae": 0.1262959986925125, + "r2": 0.6319481134414673, + "score": -0.234735553294385 + }, + "test": { + "rmse": 0.23656269916289197, + "mae": 0.15081582963466644, + "r2": 0.6175874471664429, + "score": -0.23656269916289197 + } + }, + "time": "0:02:52.457107", + "diversity": { + "train": { + "mean_centered_corr": -0.01820620521903038, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04663686826825142 + }, + "val": { + "mean_centered_corr": -0.01946132257580757, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06672783941030502 + }, + "test": { + "mean_centered_corr": -0.017525583505630493, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09117421507835388 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p00025-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p00025-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p00025-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p00025-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..bce2773ab --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p00025-greedy-heads-evaluation/0/report.json @@ -0,0 +1,226 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0p001_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22176081479492696, + "mae": 0.11819849163293839, + "r2": 0.6858164072036743, + "score": -0.22176081479492696 + }, + "val": { + "rmse": 0.23338687341364966, + "mae": 0.12646035850048065, + "r2": 0.6361653208732605, + "score": -0.23338687341364966 + }, + "test": { + "rmse": 0.23796418466778702, + "mae": 0.1546720266342163, + "r2": 0.613042950630188, + "score": -0.23796418466778702 + } + }, + "time": "0:02:52.457107", + "diversity": { + "train": { + "mean_centered_corr": -0.01820620521903038, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04663686826825142 + }, + "val": { + "mean_centered_corr": -0.01946132257580757, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06672783941030502 + }, + "test": { + "mean_centered_corr": -0.017525583505630493, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09117421507835388 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19, + 18, + 12, + 16, + 4, + 25, + 1, + 8, + 21, + 17, + 4, + 25, + 16, + 12 + ], + "head_selection_time": "0:00:00.219434" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..6d426a969 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0005-best-head-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0p001_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2378248549747135, + "mae": 0.12229524552822113, + "r2": 0.6386497020721436, + "score": -0.2378248549747135 + }, + "val": { + "rmse": 0.23686825686043878, + "mae": 0.12953220307826996, + "r2": 0.6252298355102539, + "score": -0.23686825686043878 + }, + "test": { + "rmse": 0.24350702848855615, + "mae": 0.15873141586780548, + "r2": 0.5948063731193542, + "score": -0.24350702848855615 + } + }, + "time": "0:02:41.028872", + "diversity": { + "train": { + "mean_centered_corr": -0.018550843000411987, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045891255140304565 + }, + "val": { + "mean_centered_corr": -0.019669199362397194, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06302221864461899 + }, + "test": { + "mean_centered_corr": -0.016400013118982315, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0854080319404602 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015299" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0005-evaluation/0.toml new file mode 100644 index 000000000..f2a902eb5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0005-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.0005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0005-evaluation/0/report.json new file mode 100644 index 000000000..63e961493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0005-evaluation/0/report.json @@ -0,0 +1,209 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0p001_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23087702165729493, + "mae": 0.12027005851268768, + "r2": 0.659454345703125, + "score": -0.23087702165729493 + }, + "val": { + "rmse": 0.2353183042208091, + "mae": 0.12467478215694427, + "r2": 0.6301184296607971, + "score": -0.2353183042208091 + }, + "test": { + "rmse": 0.2326811298212561, + "mae": 0.14326241612434387, + "r2": 0.6300339102745056, + "score": -0.2326811298212561 + } + }, + "time": "0:02:41.028872", + "diversity": { + "train": { + "mean_centered_corr": -0.018550843000411987, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045891255140304565 + }, + "val": { + "mean_centered_corr": -0.019669199362397194, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06302221864461899 + }, + "test": { + "mean_centered_corr": -0.016400013118982315, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0854080319404602 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..2376110b7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,220 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0p001_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.231488372724441, + "mae": 0.12102989852428436, + "r2": 0.657648503780365, + "score": -0.231488372724441 + }, + "val": { + "rmse": 0.23313960483456422, + "mae": 0.12591955065727234, + "r2": 0.636935830116272, + "score": -0.23313960483456422 + }, + "test": { + "rmse": 0.23741637847101169, + "mae": 0.1548576056957245, + "r2": 0.6148224472999573, + "score": -0.23741637847101169 + } + }, + "time": "0:02:41.028872", + "diversity": { + "train": { + "mean_centered_corr": -0.018550843000411987, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045891255140304565 + }, + "val": { + "mean_centered_corr": -0.019669199362397194, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06302221864461899 + }, + "test": { + "mean_centered_corr": -0.016400013118982315, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0854080319404602 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 0, + 19, + 28, + 18, + 16, + 25, + 21 + ], + "head_selection_time": "0:00:00.130266" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..e7fec36f1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p001-best-head-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0p001_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2349271548648021, + "mae": 0.13177132606506348, + "r2": 0.6474015712738037, + "score": -0.2349271548648021 + }, + "val": { + "rmse": 0.23779476251258524, + "mae": 0.12809278070926666, + "r2": 0.6222922801971436, + "score": -0.23779476251258524 + }, + "test": { + "rmse": 0.2405214145563063, + "mae": 0.14176644384860992, + "r2": 0.6046815514564514, + "score": -0.2405214145563063 + } + }, + "time": "0:02:41.928395", + "diversity": { + "train": { + "mean_centered_corr": -0.018742891028523445, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04514782875776291 + }, + "val": { + "mean_centered_corr": -0.018679512664675713, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05872314050793648 + }, + "test": { + "mean_centered_corr": -0.01721641607582569, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0779365748167038 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19 + ], + "head_selection_time": "0:00:00.015559" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p001-evaluation/0.toml new file mode 100644 index 000000000..d8465a88b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p001-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.001 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p001-evaluation/0/report.json new file mode 100644 index 000000000..5e0b1abfa --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p001-evaluation/0/report.json @@ -0,0 +1,209 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0p001_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2304459980871274, + "mae": 0.11964496970176697, + "r2": 0.6607247591018677, + "score": -0.2304459980871274 + }, + "val": { + "rmse": 0.23492211221645545, + "mae": 0.1262272447347641, + "r2": 0.6313629150390625, + "score": -0.23492211221645545 + }, + "test": { + "rmse": 0.23523946918101388, + "mae": 0.148455411195755, + "r2": 0.6218535900115967, + "score": -0.23523946918101388 + } + }, + "time": "0:02:41.928395", + "diversity": { + "train": { + "mean_centered_corr": -0.018742891028523445, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04514782875776291 + }, + "val": { + "mean_centered_corr": -0.018679512664675713, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05872314050793648 + }, + "test": { + "mean_centered_corr": -0.01721641607582569, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0779365748167038 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..224b2e6db --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,219 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0p001_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2307565556940205, + "mae": 0.12035396695137024, + "r2": 0.6598096489906311, + "score": -0.2307565556940205 + }, + "val": { + "rmse": 0.2333871447652357, + "mae": 0.12601718306541443, + "r2": 0.6361644268035889, + "score": -0.2333871447652357 + }, + "test": { + "rmse": 0.2345517842823023, + "mae": 0.14844825863838196, + "r2": 0.6240612864494324, + "score": -0.2345517842823023 + } + }, + "time": "0:02:41.928395", + "diversity": { + "train": { + "mean_centered_corr": -0.018742891028523445, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04514782875776291 + }, + "val": { + "mean_centered_corr": -0.018679512664675713, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05872314050793648 + }, + "test": { + "mean_centered_corr": -0.01721641607582569, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0779365748167038 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19, + 0, + 4, + 12, + 28, + 18, + 24 + ], + "head_selection_time": "0:00:00.118122" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0025-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0025-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0025-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0025-best-head-evaluation/0/report.json new file mode 100644 index 000000000..19efdf1b2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0025-best-head-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0p001_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2304673922223974, + "mae": 0.13124814629554749, + "r2": 0.6606616973876953, + "score": -0.2304673922223974 + }, + "val": { + "rmse": 0.2389336160662929, + "mae": 0.1372906118631363, + "r2": 0.6186658143997192, + "score": -0.2389336160662929 + }, + "test": { + "rmse": 0.2358168359752649, + "mae": 0.1429797112941742, + "r2": 0.6199951171875, + "score": -0.2358168359752649 + } + }, + "time": "0:02:54.368253", + "diversity": { + "train": { + "mean_centered_corr": -0.018927747383713722, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04583616182208061 + }, + "val": { + "mean_centered_corr": -0.018819812685251236, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06454488635063171 + }, + "test": { + "mean_centered_corr": -0.014754452742636204, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09140031039714813 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 16 + ], + "head_selection_time": "0:00:00.015372" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0025-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0025-evaluation/0.toml new file mode 100644 index 000000000..f0c088e21 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0025-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.0025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0025-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0025-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0025-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0025-evaluation/0/report.json new file mode 100644 index 000000000..14df0c853 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0025-evaluation/0/report.json @@ -0,0 +1,209 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0p001_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22191430166883294, + "mae": 0.11762423813343048, + "r2": 0.6853813529014587, + "score": -0.22191430166883294 + }, + "val": { + "rmse": 0.23496063498765302, + "mae": 0.12751367688179016, + "r2": 0.6312419772148132, + "score": -0.23496063498765302 + }, + "test": { + "rmse": 0.2373818009282627, + "mae": 0.15140388906002045, + "r2": 0.6149346828460693, + "score": -0.2373818009282627 + } + }, + "time": "0:02:54.368253", + "diversity": { + "train": { + "mean_centered_corr": -0.018927747383713722, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04583616182208061 + }, + "val": { + "mean_centered_corr": -0.018819812685251236, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06454488635063171 + }, + "test": { + "mean_centered_corr": -0.014754452742636204, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09140031039714813 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0025-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0025-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0025-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0025-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..3c98033c6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0025-greedy-heads-evaluation/0/report.json @@ -0,0 +1,221 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0p001_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22186501776040204, + "mae": 0.11837654560804367, + "r2": 0.685521125793457, + "score": -0.22186501776040204 + }, + "val": { + "rmse": 0.23389711137908714, + "mae": 0.1278168261051178, + "r2": 0.6345727443695068, + "score": -0.23389711137908714 + }, + "test": { + "rmse": 0.2407960992475978, + "mae": 0.156558558344841, + "r2": 0.6037781238555908, + "score": -0.2407960992475978 + } + }, + "time": "0:02:54.368253", + "diversity": { + "train": { + "mean_centered_corr": -0.018927747383713722, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04583616182208061 + }, + "val": { + "mean_centered_corr": -0.018819812685251236, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06454488635063171 + }, + "test": { + "mean_centered_corr": -0.014754452742636204, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09140031039714813 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 16, + 19, + 25, + 12, + 18, + 4, + 13, + 31, + 22 + ], + "head_selection_time": "0:00:00.144189" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p00025-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p00025-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p00025-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p00025-best-head-evaluation/0/report.json new file mode 100644 index 000000000..2a1d2c29f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p00025-best-head-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n1em05_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22704868110587859, + "mae": 0.13059158623218536, + "r2": 0.6706544160842896, + "score": -0.22704868110587859 + }, + "val": { + "rmse": 0.23842471915921798, + "mae": 0.13392238318920135, + "r2": 0.6202884912490845, + "score": -0.23842471915921798 + }, + "test": { + "rmse": 0.34030246469362907, + "mae": 0.26470643281936646, + "r2": 0.20864784717559814, + "score": -0.34030246469362907 + } + }, + "time": "0:02:53.902694", + "diversity": { + "train": { + "mean_centered_corr": -0.01857772469520569, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04806574434041977 + }, + "val": { + "mean_centered_corr": -0.018434280529618263, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06849802285432816 + }, + "test": { + "mean_centered_corr": -0.01695123501121998, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09515117108821869 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25 + ], + "head_selection_time": "0:00:00.015076" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p00025-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p00025-evaluation/0.toml new file mode 100644 index 000000000..76382e745 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p00025-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.00025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p00025-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p00025-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p00025-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p00025-evaluation/0/report.json new file mode 100644 index 000000000..f496496c9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p00025-evaluation/0/report.json @@ -0,0 +1,209 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n1em05_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22022479049650132, + "mae": 0.11967124044895172, + "r2": 0.6901537179946899, + "score": -0.22022479049650132 + }, + "val": { + "rmse": 0.23507776598207258, + "mae": 0.1284610629081726, + "r2": 0.6308742761611938, + "score": -0.23507776598207258 + }, + "test": { + "rmse": 0.2438242655494171, + "mae": 0.16113010048866272, + "r2": 0.5937499403953552, + "score": -0.2438242655494171 + } + }, + "time": "0:02:53.902694", + "diversity": { + "train": { + "mean_centered_corr": -0.01857772469520569, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04806574434041977 + }, + "val": { + "mean_centered_corr": -0.018434280529618263, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06849802285432816 + }, + "test": { + "mean_centered_corr": -0.01695123501121998, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09515117108821869 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p00025-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p00025-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p00025-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p00025-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..aa6c9a0ac --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p00025-greedy-heads-evaluation/0/report.json @@ -0,0 +1,218 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n1em05_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22148010747656677, + "mae": 0.12138405442237854, + "r2": 0.6866112947463989, + "score": -0.22148010747656677 + }, + "val": { + "rmse": 0.23367244804564585, + "mae": 0.127582848072052, + "r2": 0.6352744102478027, + "score": -0.23367244804564585 + }, + "test": { + "rmse": 0.25122008031651116, + "mae": 0.17190353572368622, + "r2": 0.5687309503555298, + "score": -0.25122008031651116 + } + }, + "time": "0:02:53.902694", + "diversity": { + "train": { + "mean_centered_corr": -0.01857772469520569, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04806574434041977 + }, + "val": { + "mean_centered_corr": -0.018434280529618263, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06849802285432816 + }, + "test": { + "mean_centered_corr": -0.01695123501121998, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09515117108821869 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25, + 19, + 4, + 21, + 29, + 20 + ], + "head_selection_time": "0:00:00.100741" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..e11c2f4c4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0005-best-head-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n1em05_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2383916630314474, + "mae": 0.12477622181177139, + "r2": 0.6369252800941467, + "score": -0.2383916630314474 + }, + "val": { + "rmse": 0.23743311228743408, + "mae": 0.13437791168689728, + "r2": 0.6234402656555176, + "score": -0.23743311228743408 + }, + "test": { + "rmse": 0.24918858157196577, + "mae": 0.16689999401569366, + "r2": 0.5756776332855225, + "score": -0.24918858157196577 + } + }, + "time": "0:02:39.963879", + "diversity": { + "train": { + "mean_centered_corr": -0.018939347937703133, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04571089893579483 + }, + "val": { + "mean_centered_corr": -0.01955636776983738, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06383588165044785 + }, + "test": { + "mean_centered_corr": -0.01509108766913414, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08491861075162888 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015353" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0005-evaluation/0.toml new file mode 100644 index 000000000..91a0b5da7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0005-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.0005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0005-evaluation/0/report.json new file mode 100644 index 000000000..cd9172ab2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0005-evaluation/0/report.json @@ -0,0 +1,209 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n1em05_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23062436459099714, + "mae": 0.12175119668245316, + "r2": 0.6601992845535278, + "score": -0.23062436459099714 + }, + "val": { + "rmse": 0.23477109182279635, + "mae": 0.12639090418815613, + "r2": 0.6318367123603821, + "score": -0.23477109182279635 + }, + "test": { + "rmse": 0.2340687021877646, + "mae": 0.14674966037273407, + "r2": 0.6256082057952881, + "score": -0.2340687021877646 + } + }, + "time": "0:02:39.963879", + "diversity": { + "train": { + "mean_centered_corr": -0.018939347937703133, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04571089893579483 + }, + "val": { + "mean_centered_corr": -0.01955636776983738, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06383588165044785 + }, + "test": { + "mean_centered_corr": -0.01509108766913414, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08491861075162888 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..9497066b8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,224 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n1em05_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23107210482218127, + "mae": 0.12221947312355042, + "r2": 0.6588786244392395, + "score": -0.23107210482218127 + }, + "val": { + "rmse": 0.23308539046397073, + "mae": 0.12705643475055695, + "r2": 0.6371046900749207, + "score": -0.23308539046397073 + }, + "test": { + "rmse": 0.23474214725238218, + "mae": 0.15055741369724274, + "r2": 0.623450756072998, + "score": -0.23474214725238218 + } + }, + "time": "0:02:39.963879", + "diversity": { + "train": { + "mean_centered_corr": -0.018939347937703133, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04571089893579483 + }, + "val": { + "mean_centered_corr": -0.01955636776983738, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06383588165044785 + }, + "test": { + "mean_centered_corr": -0.01509108766913414, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08491861075162888 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 18, + 19, + 21, + 12, + 20, + 18, + 16, + 2, + 0, + 10, + 18 + ], + "head_selection_time": "0:00:00.191793" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..37f40c1de --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p001-best-head-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n1em05_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2255741875961414, + "mae": 0.11985473334789276, + "r2": 0.6749181747436523, + "score": -0.2255741875961414 + }, + "val": { + "rmse": 0.23707897142181117, + "mae": 0.13087449967861176, + "r2": 0.6245627999305725, + "score": -0.23707897142181117 + }, + "test": { + "rmse": 0.23966200804896912, + "mae": 0.15106384456157684, + "r2": 0.6075015068054199, + "score": -0.23966200804896912 + } + }, + "time": "0:02:54.345854", + "diversity": { + "train": { + "mean_centered_corr": -0.018778949975967407, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045425642281770706 + }, + "val": { + "mean_centered_corr": -0.019775288179516792, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06173905357718468 + }, + "test": { + "mean_centered_corr": -0.017589790746569633, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0850314125418663 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 12 + ], + "head_selection_time": "0:00:00.015441" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p001-evaluation/0.toml new file mode 100644 index 000000000..22cb9039a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p001-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.001 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p001-evaluation/0/report.json new file mode 100644 index 000000000..95dc29351 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p001-evaluation/0/report.json @@ -0,0 +1,209 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n1em05_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22242528246194596, + "mae": 0.11885363608598709, + "r2": 0.6839307546615601, + "score": -0.22242528246194596 + }, + "val": { + "rmse": 0.23488388446999084, + "mae": 0.1276719719171524, + "r2": 0.6314828395843506, + "score": -0.23488388446999084 + }, + "test": { + "rmse": 0.23845471649958624, + "mae": 0.1534513682126999, + "r2": 0.611445963382721, + "score": -0.23845471649958624 + } + }, + "time": "0:02:54.345854", + "diversity": { + "train": { + "mean_centered_corr": -0.018778949975967407, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045425642281770706 + }, + "val": { + "mean_centered_corr": -0.019775288179516792, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06173905357718468 + }, + "test": { + "mean_centered_corr": -0.017589790746569633, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0850314125418663 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..ca69ed85d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,217 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n1em05_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22164964682417307, + "mae": 0.11660284548997879, + "r2": 0.6861313581466675, + "score": -0.22164964682417307 + }, + "val": { + "rmse": 0.23314020403866242, + "mae": 0.126881942152977, + "r2": 0.6369339823722839, + "score": -0.23314020403866242 + }, + "test": { + "rmse": 0.23450263045082137, + "mae": 0.14819909632205963, + "r2": 0.6242188215255737, + "score": -0.23450263045082137 + } + }, + "time": "0:02:54.345854", + "diversity": { + "train": { + "mean_centered_corr": -0.018778949975967407, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045425642281770706 + }, + "val": { + "mean_centered_corr": -0.019775288179516792, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06173905357718468 + }, + "test": { + "mean_centered_corr": -0.017589790746569633, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0850314125418663 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 12, + 18, + 19, + 1, + 4 + ], + "head_selection_time": "0:00:00.088091" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0025-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0025-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0025-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0025-best-head-evaluation/0/report.json new file mode 100644 index 000000000..fd4e6f1b5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0025-best-head-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n1em05_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2313265193817783, + "mae": 0.1316187083721161, + "r2": 0.6581270694732666, + "score": -0.2313265193817783 + }, + "val": { + "rmse": 0.2380240960109698, + "mae": 0.13445262610912323, + "r2": 0.6215634346008301, + "score": -0.2380240960109698 + }, + "test": { + "rmse": 0.23356247568168645, + "mae": 0.14156240224838257, + "r2": 0.6272258758544922, + "score": -0.23356247568168645 + } + }, + "time": "0:02:53.865255", + "diversity": { + "train": { + "mean_centered_corr": -0.0185103602707386, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0452117882668972 + }, + "val": { + "mean_centered_corr": -0.018912050873041153, + "mean_pairwise_disagreement": NaN, + "member_std": 0.066463403403759 + }, + "test": { + "mean_centered_corr": -0.01741606369614601, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09375505894422531 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 16 + ], + "head_selection_time": "0:00:00.015257" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0025-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0025-evaluation/0.toml new file mode 100644 index 000000000..79a2d960c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0025-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.0025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0025-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0025-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0025-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0025-evaluation/0/report.json new file mode 100644 index 000000000..7b3b8425c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0025-evaluation/0/report.json @@ -0,0 +1,209 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n1em05_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22278805228101048, + "mae": 0.11789201945066452, + "r2": 0.6828989386558533, + "score": -0.22278805228101048 + }, + "val": { + "rmse": 0.23480782280356366, + "mae": 0.12755723297595978, + "r2": 0.6317214965820312, + "score": -0.23480782280356366 + }, + "test": { + "rmse": 0.23834319169461882, + "mae": 0.15305806696414948, + "r2": 0.6118093729019165, + "score": -0.23834319169461882 + } + }, + "time": "0:02:53.865255", + "diversity": { + "train": { + "mean_centered_corr": -0.0185103602707386, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0452117882668972 + }, + "val": { + "mean_centered_corr": -0.018912050873041153, + "mean_pairwise_disagreement": NaN, + "member_std": 0.066463403403759 + }, + "test": { + "mean_centered_corr": -0.01741606369614601, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09375505894422531 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0025-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0025-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0025-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0025-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..df15ecfdf --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0025-greedy-heads-evaluation/0/report.json @@ -0,0 +1,219 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n1em05_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2236405333806169, + "mae": 0.11948943138122559, + "r2": 0.6804676055908203, + "score": -0.2236405333806169 + }, + "val": { + "rmse": 0.23277917213756988, + "mae": 0.12665784358978271, + "r2": 0.6380575895309448, + "score": -0.23277917213756988 + }, + "test": { + "rmse": 0.2432190859636802, + "mae": 0.16015446186065674, + "r2": 0.5957640409469604, + "score": -0.2432190859636802 + } + }, + "time": "0:02:53.865255", + "diversity": { + "train": { + "mean_centered_corr": -0.0185103602707386, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0452117882668972 + }, + "val": { + "mean_centered_corr": -0.018912050873041153, + "mean_pairwise_disagreement": NaN, + "member_std": 0.066463403403759 + }, + "test": { + "mean_centered_corr": -0.01741606369614601, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09375505894422531 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 16, + 21, + 4, + 22, + 19, + 25, + 7 + ], + "head_selection_time": "0:00:00.116565" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..883d73820 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2314480246211949, + "mae": 0.12086600810289383, + "r2": 0.6577678322792053, + "score": -0.2314480246211949 + }, + "val": { + "rmse": 0.23862914188251422, + "mae": 0.13088369369506836, + "r2": 0.6196370124816895, + "score": -0.23862914188251422 + }, + "test": { + "rmse": 0.23553986357063933, + "mae": 0.14342397451400757, + "r2": 0.6208872199058533, + "score": -0.23553986357063933 + } + }, + "time": "0:01:16.306157", + "diversity": { + "train": { + "mean_centered_corr": -0.019465479999780655, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04585251957178116 + }, + "val": { + "mean_centered_corr": -0.01962343417108059, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06466832011938095 + }, + "test": { + "mean_centered_corr": -0.016597073525190353, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08519241958856583 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015051" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p001-evaluation/0.toml new file mode 100644 index 000000000..c0216f239 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p001-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.95 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..188a924d4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p001-evaluation/0/report.json @@ -0,0 +1,684 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22484523994052877, + "mae": 0.1174779012799263, + "r2": 0.6770157814025879, + "score": -0.22484523994052877 + }, + "val": { + "rmse": 0.2347666170730829, + "mae": 0.1269145905971527, + "r2": 0.6318507194519043, + "score": -0.2347666170730829 + }, + "test": { + "rmse": 0.23399684917333807, + "mae": 0.14828696846961975, + "r2": 0.625838041305542, + "score": -0.23399684917333807 + } + }, + "time": "0:01:16.306157", + "diversity": { + "train": { + "mean_centered_corr": -0.019465479999780655, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04585251957178116 + }, + "val": { + "mean_centered_corr": -0.01962343417108059, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06466832011938095 + }, + "test": { + "mean_centered_corr": -0.016597073525190353, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08519241958856583 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..0b3bb4881 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,691 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22569541421502587, + "mae": 0.11816450208425522, + "r2": 0.6745686531066895, + "score": -0.22569541421502587 + }, + "val": { + "rmse": 0.23305490182627153, + "mae": 0.1270279735326767, + "r2": 0.6371996402740479, + "score": -0.23305490182627153 + }, + "test": { + "rmse": 0.23732565171059752, + "mae": 0.15259529650211334, + "r2": 0.6151168346405029, + "score": -0.23732565171059752 + } + }, + "time": "0:01:16.306157", + "diversity": { + "train": { + "mean_centered_corr": -0.019465479999780655, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04585251957178116 + }, + "val": { + "mean_centered_corr": -0.01962343417108059, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06466832011938095 + }, + "test": { + "mean_centered_corr": -0.016597073525190353, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08519241958856583 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 25, + 21, + 29 + ], + "head_selection_time": "0:00:00.072510" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..675c9212e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2241888421744036, + "mae": 0.1289592683315277, + "r2": 0.678898811340332, + "score": -0.2241888421744036 + }, + "val": { + "rmse": 0.23918568828057105, + "mae": 0.14043618738651276, + "r2": 0.6178607940673828, + "score": -0.23918568828057105 + }, + "test": { + "rmse": 0.25293341379845147, + "mae": 0.17517992854118347, + "r2": 0.5628283023834229, + "score": -0.25293341379845147 + } + }, + "time": "0:01:18.920555", + "diversity": { + "train": { + "mean_centered_corr": -0.017244398593902588, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045789685100317 + }, + "val": { + "mean_centered_corr": -0.019243216142058372, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06452018767595291 + }, + "test": { + "mean_centered_corr": -0.017609506845474243, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08646101504564285 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21 + ], + "head_selection_time": "0:00:00.015213" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p005-evaluation/0.toml new file mode 100644 index 000000000..5c1e3b792 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p005-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.95 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..8707168a8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p005-evaluation/0/report.json @@ -0,0 +1,684 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.220642866499973, + "mae": 0.11938578635454178, + "r2": 0.6889761686325073, + "score": -0.220642866499973 + }, + "val": { + "rmse": 0.2351252944190024, + "mae": 0.12928679585456848, + "r2": 0.6307249069213867, + "score": -0.2351252944190024 + }, + "test": { + "rmse": 0.23697534320527558, + "mae": 0.15434296429157257, + "r2": 0.6162521839141846, + "score": -0.23697534320527558 + } + }, + "time": "0:01:18.920555", + "diversity": { + "train": { + "mean_centered_corr": -0.017244398593902588, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045789685100317 + }, + "val": { + "mean_centered_corr": -0.019243216142058372, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06452018767595291 + }, + "test": { + "mean_centered_corr": -0.017609506845474243, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08646101504564285 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..34be5ef29 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,693 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2206414904658828, + "mae": 0.12224060297012329, + "r2": 0.6889800429344177, + "score": -0.2206414904658828 + }, + "val": { + "rmse": 0.2331196304833006, + "mae": 0.12830887734889984, + "r2": 0.6369980573654175, + "score": -0.2331196304833006 + }, + "test": { + "rmse": 0.23280496841127477, + "mae": 0.14530567824840546, + "r2": 0.6296399831771851, + "score": -0.23280496841127477 + } + }, + "time": "0:01:18.920555", + "diversity": { + "train": { + "mean_centered_corr": -0.017244398593902588, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045789685100317 + }, + "val": { + "mean_centered_corr": -0.019243216142058372, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06452018767595291 + }, + "test": { + "mean_centered_corr": -0.017609506845474243, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08646101504564285 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21, + 8, + 20, + 31, + 19, + 0 + ], + "head_selection_time": "0:00:00.102311" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..5be2251b0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2425666348719968, + "mae": 0.12444926798343658, + "r2": 0.6240968108177185, + "score": -0.2425666348719968 + }, + "val": { + "rmse": 0.23756948853650636, + "mae": 0.13087332248687744, + "r2": 0.6230076551437378, + "score": -0.23756948853650636 + }, + "test": { + "rmse": 0.24224213014777235, + "mae": 0.15561068058013916, + "r2": 0.5990049839019775, + "score": -0.24224213014777235 + } + }, + "time": "0:01:10.287046", + "diversity": { + "train": { + "mean_centered_corr": -0.019360000267624855, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04683505743741989 + }, + "val": { + "mean_centered_corr": -0.019574038684368134, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06655079871416092 + }, + "test": { + "mean_centered_corr": -0.01754024624824524, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08368789404630661 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015481" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p01-evaluation/0.toml new file mode 100644 index 000000000..c0b3f5a8d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p01-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.95 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..ad67df3e9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p01-evaluation/0/report.json @@ -0,0 +1,684 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23343200123041558, + "mae": 0.12027841061353683, + "r2": 0.6518754363059998, + "score": -0.23343200123041558 + }, + "val": { + "rmse": 0.23512438339576403, + "mae": 0.12516294419765472, + "r2": 0.6307278275489807, + "score": -0.23512438339576403 + }, + "test": { + "rmse": 0.2316782780837399, + "mae": 0.14233005046844482, + "r2": 0.633216142654419, + "score": -0.2316782780837399 + } + }, + "time": "0:01:10.287046", + "diversity": { + "train": { + "mean_centered_corr": -0.019360000267624855, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04683505743741989 + }, + "val": { + "mean_centered_corr": -0.019574038684368134, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06655079871416092 + }, + "test": { + "mean_centered_corr": -0.01754024624824524, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08368789404630661 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..41e4d69c6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,693 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23458808907797565, + "mae": 0.12325569242238998, + "r2": 0.648418664932251, + "score": -0.23458808907797565 + }, + "val": { + "rmse": 0.23291770554949479, + "mae": 0.1261579543352127, + "r2": 0.6376266479492188, + "score": -0.23291770554949479 + }, + "test": { + "rmse": 0.23458472246540202, + "mae": 0.14723080396652222, + "r2": 0.6239557266235352, + "score": -0.23458472246540202 + } + }, + "time": "0:01:10.287046", + "diversity": { + "train": { + "mean_centered_corr": -0.019360000267624855, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04683505743741989 + }, + "val": { + "mean_centered_corr": -0.019574038684368134, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06655079871416092 + }, + "test": { + "mean_centered_corr": -0.01754024624824524, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08368789404630661 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 25, + 21, + 6, + 19, + 0 + ], + "head_selection_time": "0:00:00.103333" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..36b5eaaf3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24039030054595545, + "mae": 0.1237230896949768, + "r2": 0.6308118104934692, + "score": -0.24039030054595545 + }, + "val": { + "rmse": 0.23560357747729638, + "mae": 0.13029368221759796, + "r2": 0.6292210817337036, + "score": -0.23560357747729638 + }, + "test": { + "rmse": 0.2389191702722735, + "mae": 0.15265432000160217, + "r2": 0.6099308729171753, + "score": -0.2389191702722735 + } + }, + "time": "0:01:08.862892", + "diversity": { + "train": { + "mean_centered_corr": -0.018871087580919266, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04625045508146286 + }, + "val": { + "mean_centered_corr": -0.019568709656596184, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0614706352353096 + }, + "test": { + "mean_centered_corr": -0.018940167501568794, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08073103427886963 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015418" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/0.toml new file mode 100644 index 000000000..3417f3906 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..19c46df9c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/0/report.json @@ -0,0 +1,684 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23278188472143527, + "mae": 0.12037675827741623, + "r2": 0.6538118124008179, + "score": -0.23278188472143527 + }, + "val": { + "rmse": 0.235050071622046, + "mae": 0.12535566091537476, + "r2": 0.6309611797332764, + "score": -0.235050071622046 + }, + "test": { + "rmse": 0.23184680318718037, + "mae": 0.14313000440597534, + "r2": 0.6326823234558105, + "score": -0.23184680318718037 + } + }, + "time": "0:01:08.862892", + "diversity": { + "train": { + "mean_centered_corr": -0.018871087580919266, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04625045508146286 + }, + "val": { + "mean_centered_corr": -0.019568709656596184, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0614706352353096 + }, + "test": { + "mean_centered_corr": -0.018940167501568794, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08073103427886963 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..ec8575dd0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,692 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23573796221874224, + "mae": 0.12117554992437363, + "r2": 0.6449635028839111, + "score": -0.23573796221874224 + }, + "val": { + "rmse": 0.2324412121462238, + "mae": 0.12522703409194946, + "r2": 0.6391078233718872, + "score": -0.2324412121462238 + }, + "test": { + "rmse": 0.23820740696857753, + "mae": 0.1556985229253769, + "r2": 0.6122515201568604, + "score": -0.23820740696857753 + } + }, + "time": "0:01:08.862892", + "diversity": { + "train": { + "mean_centered_corr": -0.018871087580919266, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04625045508146286 + }, + "val": { + "mean_centered_corr": -0.019568709656596184, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0614706352353096 + }, + "test": { + "mean_centered_corr": -0.018940167501568794, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08073103427886963 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 0, + 1, + 25, + 21 + ], + "head_selection_time": "0:00:00.088435" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..88521f57d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24104706607058576, + "mae": 0.1307125687599182, + "r2": 0.6287918090820312, + "score": -0.24104706607058576 + }, + "val": { + "rmse": 0.2372450969584346, + "mae": 0.1306145042181015, + "r2": 0.624036431312561, + "score": -0.2372450969584346 + }, + "test": { + "rmse": 0.23726047684878263, + "mae": 0.14689351618289948, + "r2": 0.6153281927108765, + "score": -0.23726047684878263 + } + }, + "time": "0:01:10.655002", + "diversity": { + "train": { + "mean_centered_corr": -0.019136181101202965, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04613374546170235 + }, + "val": { + "mean_centered_corr": -0.01984967105090618, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06134215369820595 + }, + "test": { + "mean_centered_corr": -0.0187427569180727, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08017097413539886 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0 + ], + "head_selection_time": "0:00:00.015354" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p005-evaluation/0.toml new file mode 100644 index 000000000..5b20437d1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p005-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..78f62d5da --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p005-evaluation/0/report.json @@ -0,0 +1,684 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23274665867971067, + "mae": 0.1198442280292511, + "r2": 0.653916597366333, + "score": -0.23274665867971067 + }, + "val": { + "rmse": 0.23505585640746507, + "mae": 0.12548410892486572, + "r2": 0.6309430599212646, + "score": -0.23505585640746507 + }, + "test": { + "rmse": 0.23276235981389973, + "mae": 0.14522086083889008, + "r2": 0.6297755241394043, + "score": -0.23276235981389973 + } + }, + "time": "0:01:10.655002", + "diversity": { + "train": { + "mean_centered_corr": -0.019136181101202965, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04613374546170235 + }, + "val": { + "mean_centered_corr": -0.01984967105090618, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06134215369820595 + }, + "test": { + "mean_centered_corr": -0.0187427569180727, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08017097413539886 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..8eed36788 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,696 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23595696457409665, + "mae": 0.12094593048095703, + "r2": 0.644303560256958, + "score": -0.23595696457409665 + }, + "val": { + "rmse": 0.2328726380397863, + "mae": 0.12572719156742096, + "r2": 0.6377668380737305, + "score": -0.2328726380397863 + }, + "test": { + "rmse": 0.23808236498886684, + "mae": 0.15540435910224915, + "r2": 0.6126585006713867, + "score": -0.23808236498886684 + } + }, + "time": "0:01:10.655002", + "diversity": { + "train": { + "mean_centered_corr": -0.019136181101202965, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04613374546170235 + }, + "val": { + "mean_centered_corr": -0.01984967105090618, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06134215369820595 + }, + "test": { + "mean_centered_corr": -0.0187427569180727, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08017097413539886 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0, + 4, + 16, + 1, + 10, + 25, + 21, + 16, + 4 + ], + "head_selection_time": "0:00:00.146234" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..7ceb5b4b6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24078892072406902, + "mae": 0.12140180170536041, + "r2": 0.6295864582061768, + "score": -0.24078892072406902 + }, + "val": { + "rmse": 0.23656618722639788, + "mae": 0.1301935315132141, + "r2": 0.6261851191520691, + "score": -0.23656618722639788 + }, + "test": { + "rmse": 0.2443526301182123, + "mae": 0.15885169804096222, + "r2": 0.5919873714447021, + "score": -0.2443526301182123 + } + }, + "time": "0:01:09.764472", + "diversity": { + "train": { + "mean_centered_corr": -0.019902383908629417, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04721210524439812 + }, + "val": { + "mean_centered_corr": -0.019949622452259064, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06361114978790283 + }, + "test": { + "mean_centered_corr": -0.019116314128041267, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07747960835695267 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015310" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p01-evaluation/0.toml new file mode 100644 index 000000000..3040481ce --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p01-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..a4a16853f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p01-evaluation/0/report.json @@ -0,0 +1,684 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23225320578180678, + "mae": 0.12026127427816391, + "r2": 0.655382513999939, + "score": -0.23225320578180678 + }, + "val": { + "rmse": 0.23504784483792887, + "mae": 0.12635678052902222, + "r2": 0.6309682130813599, + "score": -0.23504784483792887 + }, + "test": { + "rmse": 0.23465037063607563, + "mae": 0.14887936413288116, + "r2": 0.6237452030181885, + "score": -0.23465037063607563 + } + }, + "time": "0:01:09.764472", + "diversity": { + "train": { + "mean_centered_corr": -0.019902383908629417, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04721210524439812 + }, + "val": { + "mean_centered_corr": -0.019949622452259064, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06361114978790283 + }, + "test": { + "mean_centered_corr": -0.019116314128041267, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07747960835695267 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..5829f6ed4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,695 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2344874747922904, + "mae": 0.1211647242307663, + "r2": 0.6487202048301697, + "score": -0.2344874747922904 + }, + "val": { + "rmse": 0.23334387616340208, + "mae": 0.12681756913661957, + "r2": 0.6362993717193604, + "score": -0.23334387616340208 + }, + "test": { + "rmse": 0.2380340654227449, + "mae": 0.15519878268241882, + "r2": 0.6128156185150146, + "score": -0.2380340654227449 + } + }, + "time": "0:01:09.764472", + "diversity": { + "train": { + "mean_centered_corr": -0.019902383908629417, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04721210524439812 + }, + "val": { + "mean_centered_corr": -0.019949622452259064, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06361114978790283 + }, + "test": { + "mean_centered_corr": -0.019116314128041267, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07747960835695267 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 29, + 19, + 1, + 10, + 25, + 21, + 16 + ], + "head_selection_time": "0:00:00.131498" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..f83824fc1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24066084712400115, + "mae": 0.1302179992198944, + "r2": 0.6299803256988525, + "score": -0.24066084712400115 + }, + "val": { + "rmse": 0.23897181954001903, + "mae": 0.13393160700798035, + "r2": 0.6185438632965088, + "score": -0.23897181954001903 + }, + "test": { + "rmse": 0.23496733361066027, + "mae": 0.1394679695367813, + "r2": 0.6227279901504517, + "score": -0.23496733361066027 + } + }, + "time": "0:01:11.476918", + "diversity": { + "train": { + "mean_centered_corr": -0.01913526840507984, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04782585799694061 + }, + "val": { + "mean_centered_corr": -0.01699827052652836, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06600741297006607 + }, + "test": { + "mean_centered_corr": -0.020212531089782715, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08567789196968079 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 8 + ], + "head_selection_time": "0:00:00.015265" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p001-evaluation/0.toml new file mode 100644 index 000000000..425700ac8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p001-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..f76ddab00 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p001-evaluation/0/report.json @@ -0,0 +1,684 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2318619145508088, + "mae": 0.12150537222623825, + "r2": 0.6565427184104919, + "score": -0.2318619145508088 + }, + "val": { + "rmse": 0.2357328737026148, + "mae": 0.12986169755458832, + "r2": 0.6288140416145325, + "score": -0.2357328737026148 + }, + "test": { + "rmse": 0.2351772485906963, + "mae": 0.14941059052944183, + "r2": 0.6220536231994629, + "score": -0.2351772485906963 + } + }, + "time": "0:01:11.476918", + "diversity": { + "train": { + "mean_centered_corr": -0.01913526840507984, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04782585799694061 + }, + "val": { + "mean_centered_corr": -0.01699827052652836, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06600741297006607 + }, + "test": { + "mean_centered_corr": -0.020212531089782715, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08567789196968079 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..39e1ab40b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,699 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23260143300346486, + "mae": 0.12271209806203842, + "r2": 0.6543483734130859, + "score": -0.23260143300346486 + }, + "val": { + "rmse": 0.2341784288692296, + "mae": 0.1288924217224121, + "r2": 0.6336931586265564, + "score": -0.2341784288692296 + }, + "test": { + "rmse": 0.23454983071534766, + "mae": 0.14984872937202454, + "r2": 0.6240675449371338, + "score": -0.23454983071534766 + } + }, + "time": "0:01:11.476918", + "diversity": { + "train": { + "mean_centered_corr": -0.01913526840507984, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04782585799694061 + }, + "val": { + "mean_centered_corr": -0.01699827052652836, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06600741297006607 + }, + "test": { + "mean_centered_corr": -0.020212531089782715, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08567789196968079 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 8, + 21, + 27, + 19, + 3, + 25, + 4, + 12, + 0, + 18, + 3, + 24 + ], + "head_selection_time": "0:00:00.187545" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..0a3243787 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1184, + "metrics": { + "train": { + "rmse": 0.22160140516469537, + "mae": 0.12272227555513382, + "r2": 0.6862679719924927, + "score": -0.22160140516469537 + }, + "val": { + "rmse": 0.24058462987099405, + "mae": 0.13246983289718628, + "r2": 0.613377571105957, + "score": -0.24058462987099405 + }, + "test": { + "rmse": 0.2859647081281448, + "mae": 0.2032262533903122, + "r2": 0.44118958711624146, + "score": -0.2859647081281448 + } + }, + "time": "0:01:31.771928", + "diversity": { + "train": { + "mean_centered_corr": -0.0177434291690588, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05118265748023987 + }, + "val": { + "mean_centered_corr": -0.01485136803239584, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0833396464586258 + }, + "test": { + "mean_centered_corr": -0.015522093512117863, + "mean_pairwise_disagreement": NaN, + "member_std": 0.11537164449691772 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 10 + ], + "head_selection_time": "0:00:00.015141" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p005-evaluation/0.toml new file mode 100644 index 000000000..a62285992 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p005-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..a866deda9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p005-evaluation/0/report.json @@ -0,0 +1,684 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1184, + "metrics": { + "train": { + "rmse": 0.2047007403568582, + "mae": 0.11088933050632477, + "r2": 0.7322973012924194, + "score": -0.2047007403568582 + }, + "val": { + "rmse": 0.2351519581095747, + "mae": 0.12822481989860535, + "r2": 0.6306412220001221, + "score": -0.2351519581095747 + }, + "test": { + "rmse": 0.2408751648791966, + "mae": 0.15764909982681274, + "r2": 0.6035178899765015, + "score": -0.2408751648791966 + } + }, + "time": "0:01:31.771928", + "diversity": { + "train": { + "mean_centered_corr": -0.0177434291690588, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05118265748023987 + }, + "val": { + "mean_centered_corr": -0.01485136803239584, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0833396464586258 + }, + "test": { + "mean_centered_corr": -0.015522093512117863, + "mean_pairwise_disagreement": NaN, + "member_std": 0.11537164449691772 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..ece2cef9e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,705 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1184, + "metrics": { + "train": { + "rmse": 0.20677708372923248, + "mae": 0.11196121573448181, + "r2": 0.7268389463424683, + "score": -0.20677708372923248 + }, + "val": { + "rmse": 0.23316436266516308, + "mae": 0.12652936577796936, + "r2": 0.6368587017059326, + "score": -0.23316436266516308 + }, + "test": { + "rmse": 0.23171057167720172, + "mae": 0.14218251407146454, + "r2": 0.6331138610839844, + "score": -0.23171057167720172 + } + }, + "time": "0:01:31.771928", + "diversity": { + "train": { + "mean_centered_corr": -0.0177434291690588, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05118265748023987 + }, + "val": { + "mean_centered_corr": -0.01485136803239584, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0833396464586258 + }, + "test": { + "mean_centered_corr": -0.015522093512117863, + "mean_pairwise_disagreement": NaN, + "member_std": 0.11537164449691772 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 10, + 27, + 7, + 1, + 23, + 16, + 29, + 5, + 17, + 7, + 25, + 29, + 20, + 27, + 1, + 21, + 7, + 22 + ], + "head_selection_time": "0:00:00.281144" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..504a3c981 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2438513529798656, + "mae": 0.12642574310302734, + "r2": 0.6201044321060181, + "score": -0.2438513529798656 + }, + "val": { + "rmse": 0.23781902006036262, + "mae": 0.13461825251579285, + "r2": 0.6222152709960938, + "score": -0.23781902006036262 + }, + "test": { + "rmse": 0.2470883298795412, + "mae": 0.16668537259101868, + "r2": 0.5828002095222473, + "score": -0.2470883298795412 + } + }, + "time": "0:01:11.254152", + "diversity": { + "train": { + "mean_centered_corr": -0.01934780180454254, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04824277386069298 + }, + "val": { + "mean_centered_corr": -0.018993837758898735, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06548074632883072 + }, + "test": { + "mean_centered_corr": -0.019981293007731438, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0883246660232544 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015371" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p01-evaluation/0.toml new file mode 100644 index 000000000..9a8e1d1e5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p01-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..599624835 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p01-evaluation/0/report.json @@ -0,0 +1,684 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23279507911071862, + "mae": 0.12109043449163437, + "r2": 0.6537725925445557, + "score": -0.23279507911071862 + }, + "val": { + "rmse": 0.2353263382433216, + "mae": 0.12844589352607727, + "r2": 0.6300932168960571, + "score": -0.2353263382433216 + }, + "test": { + "rmse": 0.2351845825591179, + "mae": 0.1493937373161316, + "r2": 0.6220300197601318, + "score": -0.2351845825591179 + } + }, + "time": "0:01:11.254152", + "diversity": { + "train": { + "mean_centered_corr": -0.01934780180454254, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04824277386069298 + }, + "val": { + "mean_centered_corr": -0.018993837758898735, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06548074632883072 + }, + "test": { + "mean_centered_corr": -0.019981293007731438, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0883246660232544 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..e79972df5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,698 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23261467766548827, + "mae": 0.12115577608346939, + "r2": 0.6543089747428894, + "score": -0.23261467766548827 + }, + "val": { + "rmse": 0.2335463179080623, + "mae": 0.127370223402977, + "r2": 0.6356680393218994, + "score": -0.2335463179080623 + }, + "test": { + "rmse": 0.23522276143697368, + "mae": 0.15016508102416992, + "r2": 0.6219072937965393, + "score": -0.23522276143697368 + } + }, + "time": "0:01:11.254152", + "diversity": { + "train": { + "mean_centered_corr": -0.01934780180454254, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04824277386069298 + }, + "val": { + "mean_centered_corr": -0.018993837758898735, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06548074632883072 + }, + "test": { + "mean_centered_corr": -0.019981293007731438, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0883246660232544 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 19, + 1, + 20, + 25, + 12, + 21, + 25, + 3, + 28, + 19 + ], + "head_selection_time": "0:00:00.175756" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..9723c36d4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2314480246211949, + "mae": 0.12086600810289383, + "r2": 0.6577678322792053, + "score": -0.2314480246211949 + }, + "val": { + "rmse": 0.23862914188251422, + "mae": 0.13088369369506836, + "r2": 0.6196370124816895, + "score": -0.23862914188251422 + }, + "test": { + "rmse": 0.23553986357063933, + "mae": 0.14342397451400757, + "r2": 0.6208872199058533, + "score": -0.23553986357063933 + } + }, + "time": "0:01:15.113893", + "diversity": { + "train": { + "mean_centered_corr": -0.019465479999780655, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04585251957178116 + }, + "val": { + "mean_centered_corr": -0.01962343417108059, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06466832011938095 + }, + "test": { + "mean_centered_corr": -0.016597073525190353, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08519241958856583 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015247" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p001-evaluation/0.toml new file mode 100644 index 000000000..48ad2cfe9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p001-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.95 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..9c1d0f680 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p001-evaluation/0/report.json @@ -0,0 +1,684 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22484523994052877, + "mae": 0.1174779012799263, + "r2": 0.6770157814025879, + "score": -0.22484523994052877 + }, + "val": { + "rmse": 0.2347666170730829, + "mae": 0.1269145905971527, + "r2": 0.6318507194519043, + "score": -0.2347666170730829 + }, + "test": { + "rmse": 0.23399684917333807, + "mae": 0.14828696846961975, + "r2": 0.625838041305542, + "score": -0.23399684917333807 + } + }, + "time": "0:01:15.113893", + "diversity": { + "train": { + "mean_centered_corr": -0.019465479999780655, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04585251957178116 + }, + "val": { + "mean_centered_corr": -0.01962343417108059, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06466832011938095 + }, + "test": { + "mean_centered_corr": -0.016597073525190353, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08519241958856583 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..5df4cd600 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,691 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22569541421502587, + "mae": 0.11816450208425522, + "r2": 0.6745686531066895, + "score": -0.22569541421502587 + }, + "val": { + "rmse": 0.23305490182627153, + "mae": 0.1270279735326767, + "r2": 0.6371996402740479, + "score": -0.23305490182627153 + }, + "test": { + "rmse": 0.23732565171059752, + "mae": 0.15259529650211334, + "r2": 0.6151168346405029, + "score": -0.23732565171059752 + } + }, + "time": "0:01:15.113893", + "diversity": { + "train": { + "mean_centered_corr": -0.019465479999780655, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04585251957178116 + }, + "val": { + "mean_centered_corr": -0.01962343417108059, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06466832011938095 + }, + "test": { + "mean_centered_corr": -0.016597073525190353, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08519241958856583 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 25, + 21, + 29 + ], + "head_selection_time": "0:00:00.073868" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..ef0efb8bb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2241888421744036, + "mae": 0.1289592683315277, + "r2": 0.678898811340332, + "score": -0.2241888421744036 + }, + "val": { + "rmse": 0.23918568828057105, + "mae": 0.14043618738651276, + "r2": 0.6178607940673828, + "score": -0.23918568828057105 + }, + "test": { + "rmse": 0.25293341379845147, + "mae": 0.17517992854118347, + "r2": 0.5628283023834229, + "score": -0.25293341379845147 + } + }, + "time": "0:01:20.898855", + "diversity": { + "train": { + "mean_centered_corr": -0.017244398593902588, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045789685100317 + }, + "val": { + "mean_centered_corr": -0.019243216142058372, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06452018767595291 + }, + "test": { + "mean_centered_corr": -0.017609506845474243, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08646101504564285 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21 + ], + "head_selection_time": "0:00:00.015361" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p005-evaluation/0.toml new file mode 100644 index 000000000..8db1064c6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p005-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.95 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..eb10527ef --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p005-evaluation/0/report.json @@ -0,0 +1,684 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.220642866499973, + "mae": 0.11938578635454178, + "r2": 0.6889761686325073, + "score": -0.220642866499973 + }, + "val": { + "rmse": 0.2351252944190024, + "mae": 0.12928679585456848, + "r2": 0.6307249069213867, + "score": -0.2351252944190024 + }, + "test": { + "rmse": 0.23697534320527558, + "mae": 0.15434296429157257, + "r2": 0.6162521839141846, + "score": -0.23697534320527558 + } + }, + "time": "0:01:20.898855", + "diversity": { + "train": { + "mean_centered_corr": -0.017244398593902588, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045789685100317 + }, + "val": { + "mean_centered_corr": -0.019243216142058372, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06452018767595291 + }, + "test": { + "mean_centered_corr": -0.017609506845474243, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08646101504564285 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..24faab8e3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,693 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2206414904658828, + "mae": 0.12224060297012329, + "r2": 0.6889800429344177, + "score": -0.2206414904658828 + }, + "val": { + "rmse": 0.2331196304833006, + "mae": 0.12830887734889984, + "r2": 0.6369980573654175, + "score": -0.2331196304833006 + }, + "test": { + "rmse": 0.23280496841127477, + "mae": 0.14530567824840546, + "r2": 0.6296399831771851, + "score": -0.23280496841127477 + } + }, + "time": "0:01:20.898855", + "diversity": { + "train": { + "mean_centered_corr": -0.017244398593902588, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045789685100317 + }, + "val": { + "mean_centered_corr": -0.019243216142058372, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06452018767595291 + }, + "test": { + "mean_centered_corr": -0.017609506845474243, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08646101504564285 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21, + 8, + 20, + 31, + 19, + 0 + ], + "head_selection_time": "0:00:00.101908" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..ae99c67f7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2425666348719968, + "mae": 0.12444926798343658, + "r2": 0.6240968108177185, + "score": -0.2425666348719968 + }, + "val": { + "rmse": 0.23756948853650636, + "mae": 0.13087332248687744, + "r2": 0.6230076551437378, + "score": -0.23756948853650636 + }, + "test": { + "rmse": 0.24224213014777235, + "mae": 0.15561068058013916, + "r2": 0.5990049839019775, + "score": -0.24224213014777235 + } + }, + "time": "0:01:11.295558", + "diversity": { + "train": { + "mean_centered_corr": -0.019360000267624855, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04683505743741989 + }, + "val": { + "mean_centered_corr": -0.019574038684368134, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06655079871416092 + }, + "test": { + "mean_centered_corr": -0.01754024624824524, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08368789404630661 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015347" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p01-evaluation/0.toml new file mode 100644 index 000000000..2636255e4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p01-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.95 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..73e8ea7c3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p01-evaluation/0/report.json @@ -0,0 +1,684 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23343200123041558, + "mae": 0.12027841061353683, + "r2": 0.6518754363059998, + "score": -0.23343200123041558 + }, + "val": { + "rmse": 0.23512438339576403, + "mae": 0.12516294419765472, + "r2": 0.6307278275489807, + "score": -0.23512438339576403 + }, + "test": { + "rmse": 0.2316782780837399, + "mae": 0.14233005046844482, + "r2": 0.633216142654419, + "score": -0.2316782780837399 + } + }, + "time": "0:01:11.295558", + "diversity": { + "train": { + "mean_centered_corr": -0.019360000267624855, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04683505743741989 + }, + "val": { + "mean_centered_corr": -0.019574038684368134, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06655079871416092 + }, + "test": { + "mean_centered_corr": -0.01754024624824524, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08368789404630661 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..63be52b28 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,693 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23458808907797565, + "mae": 0.12325569242238998, + "r2": 0.648418664932251, + "score": -0.23458808907797565 + }, + "val": { + "rmse": 0.23291770554949479, + "mae": 0.1261579543352127, + "r2": 0.6376266479492188, + "score": -0.23291770554949479 + }, + "test": { + "rmse": 0.23458472246540202, + "mae": 0.14723080396652222, + "r2": 0.6239557266235352, + "score": -0.23458472246540202 + } + }, + "time": "0:01:11.295558", + "diversity": { + "train": { + "mean_centered_corr": -0.019360000267624855, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04683505743741989 + }, + "val": { + "mean_centered_corr": -0.019574038684368134, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06655079871416092 + }, + "test": { + "mean_centered_corr": -0.01754024624824524, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08368789404630661 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 25, + 21, + 6, + 19, + 0 + ], + "head_selection_time": "0:00:00.101499" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..3fbae2439 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24039030054595545, + "mae": 0.1237230896949768, + "r2": 0.6308118104934692, + "score": -0.24039030054595545 + }, + "val": { + "rmse": 0.23560357747729638, + "mae": 0.13029368221759796, + "r2": 0.6292210817337036, + "score": -0.23560357747729638 + }, + "test": { + "rmse": 0.2389191702722735, + "mae": 0.15265432000160217, + "r2": 0.6099308729171753, + "score": -0.2389191702722735 + } + }, + "time": "0:01:11.031390", + "diversity": { + "train": { + "mean_centered_corr": -0.018871087580919266, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04625045508146286 + }, + "val": { + "mean_centered_corr": -0.019568709656596184, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0614706352353096 + }, + "test": { + "mean_centered_corr": -0.018940167501568794, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08073103427886963 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015409" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p001-evaluation/0.toml new file mode 100644 index 000000000..6cf1a5f53 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p001-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..bf5dfa8ef --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p001-evaluation/0/report.json @@ -0,0 +1,684 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23278188472143527, + "mae": 0.12037675827741623, + "r2": 0.6538118124008179, + "score": -0.23278188472143527 + }, + "val": { + "rmse": 0.235050071622046, + "mae": 0.12535566091537476, + "r2": 0.6309611797332764, + "score": -0.235050071622046 + }, + "test": { + "rmse": 0.23184680318718037, + "mae": 0.14313000440597534, + "r2": 0.6326823234558105, + "score": -0.23184680318718037 + } + }, + "time": "0:01:11.031390", + "diversity": { + "train": { + "mean_centered_corr": -0.018871087580919266, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04625045508146286 + }, + "val": { + "mean_centered_corr": -0.019568709656596184, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0614706352353096 + }, + "test": { + "mean_centered_corr": -0.018940167501568794, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08073103427886963 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..134f955c4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,692 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23573796221874224, + "mae": 0.12117554992437363, + "r2": 0.6449635028839111, + "score": -0.23573796221874224 + }, + "val": { + "rmse": 0.2324412121462238, + "mae": 0.12522703409194946, + "r2": 0.6391078233718872, + "score": -0.2324412121462238 + }, + "test": { + "rmse": 0.23820740696857753, + "mae": 0.1556985229253769, + "r2": 0.6122515201568604, + "score": -0.23820740696857753 + } + }, + "time": "0:01:11.031390", + "diversity": { + "train": { + "mean_centered_corr": -0.018871087580919266, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04625045508146286 + }, + "val": { + "mean_centered_corr": -0.019568709656596184, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0614706352353096 + }, + "test": { + "mean_centered_corr": -0.018940167501568794, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08073103427886963 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 0, + 1, + 25, + 21 + ], + "head_selection_time": "0:00:00.088364" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..43a45a359 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24104706607058576, + "mae": 0.1307125687599182, + "r2": 0.6287918090820312, + "score": -0.24104706607058576 + }, + "val": { + "rmse": 0.2372450969584346, + "mae": 0.1306145042181015, + "r2": 0.624036431312561, + "score": -0.2372450969584346 + }, + "test": { + "rmse": 0.23726047684878263, + "mae": 0.14689351618289948, + "r2": 0.6153281927108765, + "score": -0.23726047684878263 + } + }, + "time": "0:01:12.101991", + "diversity": { + "train": { + "mean_centered_corr": -0.019136181101202965, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04613374546170235 + }, + "val": { + "mean_centered_corr": -0.01984967105090618, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06134215369820595 + }, + "test": { + "mean_centered_corr": -0.0187427569180727, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08017097413539886 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0 + ], + "head_selection_time": "0:00:00.015275" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p005-evaluation/0.toml new file mode 100644 index 000000000..ed5045f78 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p005-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..caca58912 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p005-evaluation/0/report.json @@ -0,0 +1,684 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23274665867971067, + "mae": 0.1198442280292511, + "r2": 0.653916597366333, + "score": -0.23274665867971067 + }, + "val": { + "rmse": 0.23505585640746507, + "mae": 0.12548410892486572, + "r2": 0.6309430599212646, + "score": -0.23505585640746507 + }, + "test": { + "rmse": 0.23276235981389973, + "mae": 0.14522086083889008, + "r2": 0.6297755241394043, + "score": -0.23276235981389973 + } + }, + "time": "0:01:12.101991", + "diversity": { + "train": { + "mean_centered_corr": -0.019136181101202965, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04613374546170235 + }, + "val": { + "mean_centered_corr": -0.01984967105090618, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06134215369820595 + }, + "test": { + "mean_centered_corr": -0.0187427569180727, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08017097413539886 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..d62e44a81 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,696 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23595696457409665, + "mae": 0.12094593048095703, + "r2": 0.644303560256958, + "score": -0.23595696457409665 + }, + "val": { + "rmse": 0.2328726380397863, + "mae": 0.12572719156742096, + "r2": 0.6377668380737305, + "score": -0.2328726380397863 + }, + "test": { + "rmse": 0.23808236498886684, + "mae": 0.15540435910224915, + "r2": 0.6126585006713867, + "score": -0.23808236498886684 + } + }, + "time": "0:01:12.101991", + "diversity": { + "train": { + "mean_centered_corr": -0.019136181101202965, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04613374546170235 + }, + "val": { + "mean_centered_corr": -0.01984967105090618, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06134215369820595 + }, + "test": { + "mean_centered_corr": -0.0187427569180727, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08017097413539886 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0, + 4, + 16, + 1, + 10, + 25, + 21, + 16, + 4 + ], + "head_selection_time": "0:00:00.146138" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..8e82cc8bc --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24078892072406902, + "mae": 0.12140180170536041, + "r2": 0.6295864582061768, + "score": -0.24078892072406902 + }, + "val": { + "rmse": 0.23656618722639788, + "mae": 0.1301935315132141, + "r2": 0.6261851191520691, + "score": -0.23656618722639788 + }, + "test": { + "rmse": 0.2443526301182123, + "mae": 0.15885169804096222, + "r2": 0.5919873714447021, + "score": -0.2443526301182123 + } + }, + "time": "0:01:10.136519", + "diversity": { + "train": { + "mean_centered_corr": -0.019902383908629417, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04721210524439812 + }, + "val": { + "mean_centered_corr": -0.019949622452259064, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06361114978790283 + }, + "test": { + "mean_centered_corr": -0.019116314128041267, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07747960835695267 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015011" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p01-evaluation/0.toml new file mode 100644 index 000000000..cc09b9d7d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p01-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..aa3dfb801 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p01-evaluation/0/report.json @@ -0,0 +1,684 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23225320578180678, + "mae": 0.12026127427816391, + "r2": 0.655382513999939, + "score": -0.23225320578180678 + }, + "val": { + "rmse": 0.23504784483792887, + "mae": 0.12635678052902222, + "r2": 0.6309682130813599, + "score": -0.23504784483792887 + }, + "test": { + "rmse": 0.23465037063607563, + "mae": 0.14887936413288116, + "r2": 0.6237452030181885, + "score": -0.23465037063607563 + } + }, + "time": "0:01:10.136519", + "diversity": { + "train": { + "mean_centered_corr": -0.019902383908629417, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04721210524439812 + }, + "val": { + "mean_centered_corr": -0.019949622452259064, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06361114978790283 + }, + "test": { + "mean_centered_corr": -0.019116314128041267, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07747960835695267 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..1ddbc24cd --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,695 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2344874747922904, + "mae": 0.1211647242307663, + "r2": 0.6487202048301697, + "score": -0.2344874747922904 + }, + "val": { + "rmse": 0.23334387616340208, + "mae": 0.12681756913661957, + "r2": 0.6362993717193604, + "score": -0.23334387616340208 + }, + "test": { + "rmse": 0.2380340654227449, + "mae": 0.15519878268241882, + "r2": 0.6128156185150146, + "score": -0.2380340654227449 + } + }, + "time": "0:01:10.136519", + "diversity": { + "train": { + "mean_centered_corr": -0.019902383908629417, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04721210524439812 + }, + "val": { + "mean_centered_corr": -0.019949622452259064, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06361114978790283 + }, + "test": { + "mean_centered_corr": -0.019116314128041267, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07747960835695267 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 29, + 19, + 1, + 10, + 25, + 21, + 16 + ], + "head_selection_time": "0:00:00.133304" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..4a592983a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24066084712400115, + "mae": 0.1302179992198944, + "r2": 0.6299803256988525, + "score": -0.24066084712400115 + }, + "val": { + "rmse": 0.23897181954001903, + "mae": 0.13393160700798035, + "r2": 0.6185438632965088, + "score": -0.23897181954001903 + }, + "test": { + "rmse": 0.23496733361066027, + "mae": 0.1394679695367813, + "r2": 0.6227279901504517, + "score": -0.23496733361066027 + } + }, + "time": "0:01:11.433192", + "diversity": { + "train": { + "mean_centered_corr": -0.01913526840507984, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04782585799694061 + }, + "val": { + "mean_centered_corr": -0.01699827052652836, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06600741297006607 + }, + "test": { + "mean_centered_corr": -0.020212531089782715, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08567789196968079 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 8 + ], + "head_selection_time": "0:00:00.016013" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p001-evaluation/0.toml new file mode 100644 index 000000000..abb0f08fd --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p001-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..13e36cb86 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p001-evaluation/0/report.json @@ -0,0 +1,684 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2318619145508088, + "mae": 0.12150537222623825, + "r2": 0.6565427184104919, + "score": -0.2318619145508088 + }, + "val": { + "rmse": 0.2357328737026148, + "mae": 0.12986169755458832, + "r2": 0.6288140416145325, + "score": -0.2357328737026148 + }, + "test": { + "rmse": 0.2351772485906963, + "mae": 0.14941059052944183, + "r2": 0.6220536231994629, + "score": -0.2351772485906963 + } + }, + "time": "0:01:11.433192", + "diversity": { + "train": { + "mean_centered_corr": -0.01913526840507984, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04782585799694061 + }, + "val": { + "mean_centered_corr": -0.01699827052652836, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06600741297006607 + }, + "test": { + "mean_centered_corr": -0.020212531089782715, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08567789196968079 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..ec510aaaf --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,699 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23260143300346486, + "mae": 0.12271209806203842, + "r2": 0.6543483734130859, + "score": -0.23260143300346486 + }, + "val": { + "rmse": 0.2341784288692296, + "mae": 0.1288924217224121, + "r2": 0.6336931586265564, + "score": -0.2341784288692296 + }, + "test": { + "rmse": 0.23454983071534766, + "mae": 0.14984872937202454, + "r2": 0.6240675449371338, + "score": -0.23454983071534766 + } + }, + "time": "0:01:11.433192", + "diversity": { + "train": { + "mean_centered_corr": -0.01913526840507984, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04782585799694061 + }, + "val": { + "mean_centered_corr": -0.01699827052652836, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06600741297006607 + }, + "test": { + "mean_centered_corr": -0.020212531089782715, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08567789196968079 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 8, + 21, + 27, + 19, + 3, + 25, + 4, + 12, + 0, + 18, + 3, + 24 + ], + "head_selection_time": "0:00:00.198152" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..0c1814f5c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1184, + "metrics": { + "train": { + "rmse": 0.22160140516469537, + "mae": 0.12272227555513382, + "r2": 0.6862679719924927, + "score": -0.22160140516469537 + }, + "val": { + "rmse": 0.24058462987099405, + "mae": 0.13246983289718628, + "r2": 0.613377571105957, + "score": -0.24058462987099405 + }, + "test": { + "rmse": 0.2859647081281448, + "mae": 0.2032262533903122, + "r2": 0.44118958711624146, + "score": -0.2859647081281448 + } + }, + "time": "0:01:30.694989", + "diversity": { + "train": { + "mean_centered_corr": -0.0177434291690588, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05118265748023987 + }, + "val": { + "mean_centered_corr": -0.01485136803239584, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0833396464586258 + }, + "test": { + "mean_centered_corr": -0.015522093512117863, + "mean_pairwise_disagreement": NaN, + "member_std": 0.11537164449691772 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 10 + ], + "head_selection_time": "0:00:00.015392" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p005-evaluation/0.toml new file mode 100644 index 000000000..7c3998233 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p005-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..0eb526983 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p005-evaluation/0/report.json @@ -0,0 +1,684 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1184, + "metrics": { + "train": { + "rmse": 0.2047007403568582, + "mae": 0.11088933050632477, + "r2": 0.7322973012924194, + "score": -0.2047007403568582 + }, + "val": { + "rmse": 0.2351519581095747, + "mae": 0.12822481989860535, + "r2": 0.6306412220001221, + "score": -0.2351519581095747 + }, + "test": { + "rmse": 0.2408751648791966, + "mae": 0.15764909982681274, + "r2": 0.6035178899765015, + "score": -0.2408751648791966 + } + }, + "time": "0:01:30.694989", + "diversity": { + "train": { + "mean_centered_corr": -0.0177434291690588, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05118265748023987 + }, + "val": { + "mean_centered_corr": -0.01485136803239584, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0833396464586258 + }, + "test": { + "mean_centered_corr": -0.015522093512117863, + "mean_pairwise_disagreement": NaN, + "member_std": 0.11537164449691772 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..9553bbb39 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,705 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1184, + "metrics": { + "train": { + "rmse": 0.20677708372923248, + "mae": 0.11196121573448181, + "r2": 0.7268389463424683, + "score": -0.20677708372923248 + }, + "val": { + "rmse": 0.23316436266516308, + "mae": 0.12652936577796936, + "r2": 0.6368587017059326, + "score": -0.23316436266516308 + }, + "test": { + "rmse": 0.23171057167720172, + "mae": 0.14218251407146454, + "r2": 0.6331138610839844, + "score": -0.23171057167720172 + } + }, + "time": "0:01:30.694989", + "diversity": { + "train": { + "mean_centered_corr": -0.0177434291690588, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05118265748023987 + }, + "val": { + "mean_centered_corr": -0.01485136803239584, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0833396464586258 + }, + "test": { + "mean_centered_corr": -0.015522093512117863, + "mean_pairwise_disagreement": NaN, + "member_std": 0.11537164449691772 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 10, + 27, + 7, + 1, + 23, + 16, + 29, + 5, + 17, + 7, + 25, + 29, + 20, + 27, + 1, + 21, + 7, + 22 + ], + "head_selection_time": "0:00:00.284355" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..513383bf7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2438513529798656, + "mae": 0.12642574310302734, + "r2": 0.6201044321060181, + "score": -0.2438513529798656 + }, + "val": { + "rmse": 0.23781902006036262, + "mae": 0.13461825251579285, + "r2": 0.6222152709960938, + "score": -0.23781902006036262 + }, + "test": { + "rmse": 0.2470883298795412, + "mae": 0.16668537259101868, + "r2": 0.5828002095222473, + "score": -0.2470883298795412 + } + }, + "time": "0:01:10.745769", + "diversity": { + "train": { + "mean_centered_corr": -0.01934780180454254, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04824277386069298 + }, + "val": { + "mean_centered_corr": -0.018993837758898735, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06548074632883072 + }, + "test": { + "mean_centered_corr": -0.019981293007731438, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0883246660232544 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015527" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p01-evaluation/0.toml new file mode 100644 index 000000000..c73212d0c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p01-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..0c3eb60a2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p01-evaluation/0/report.json @@ -0,0 +1,684 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23279507911071862, + "mae": 0.12109043449163437, + "r2": 0.6537725925445557, + "score": -0.23279507911071862 + }, + "val": { + "rmse": 0.2353263382433216, + "mae": 0.12844589352607727, + "r2": 0.6300932168960571, + "score": -0.2353263382433216 + }, + "test": { + "rmse": 0.2351845825591179, + "mae": 0.1493937373161316, + "r2": 0.6220300197601318, + "score": -0.2351845825591179 + } + }, + "time": "0:01:10.745769", + "diversity": { + "train": { + "mean_centered_corr": -0.01934780180454254, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04824277386069298 + }, + "val": { + "mean_centered_corr": -0.018993837758898735, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06548074632883072 + }, + "test": { + "mean_centered_corr": -0.019981293007731438, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0883246660232544 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..9bd901fe9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,698 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23261467766548827, + "mae": 0.12115577608346939, + "r2": 0.6543089747428894, + "score": -0.23261467766548827 + }, + "val": { + "rmse": 0.2335463179080623, + "mae": 0.127370223402977, + "r2": 0.6356680393218994, + "score": -0.2335463179080623 + }, + "test": { + "rmse": 0.23522276143697368, + "mae": 0.15016508102416992, + "r2": 0.6219072937965393, + "score": -0.23522276143697368 + } + }, + "time": "0:01:10.745769", + "diversity": { + "train": { + "mean_centered_corr": -0.01934780180454254, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04824277386069298 + }, + "val": { + "mean_centered_corr": -0.018993837758898735, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06548074632883072 + }, + "test": { + "mean_centered_corr": -0.019981293007731438, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0883246660232544 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 19, + 1, + 20, + 25, + 12, + 21, + 25, + 3, + 28, + 19 + ], + "head_selection_time": "0:00:00.177452" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..5e826e434 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2314480246211949, + "mae": 0.12086600810289383, + "r2": 0.6577678322792053, + "score": -0.2314480246211949 + }, + "val": { + "rmse": 0.23862914188251422, + "mae": 0.13088369369506836, + "r2": 0.6196370124816895, + "score": -0.23862914188251422 + }, + "test": { + "rmse": 0.23553986357063933, + "mae": 0.14342397451400757, + "r2": 0.6208872199058533, + "score": -0.23553986357063933 + } + }, + "time": "0:01:17.740928", + "diversity": { + "train": { + "mean_centered_corr": -0.019465479999780655, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04585251957178116 + }, + "val": { + "mean_centered_corr": -0.01962343417108059, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06466832011938095 + }, + "test": { + "mean_centered_corr": -0.016597073525190353, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08519241958856583 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015359" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p001-evaluation/0.toml new file mode 100644 index 000000000..c120dab19 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p001-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.95 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..76187575b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p001-evaluation/0/report.json @@ -0,0 +1,684 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22484523994052877, + "mae": 0.1174779012799263, + "r2": 0.6770157814025879, + "score": -0.22484523994052877 + }, + "val": { + "rmse": 0.2347666170730829, + "mae": 0.1269145905971527, + "r2": 0.6318507194519043, + "score": -0.2347666170730829 + }, + "test": { + "rmse": 0.23399684917333807, + "mae": 0.14828696846961975, + "r2": 0.625838041305542, + "score": -0.23399684917333807 + } + }, + "time": "0:01:17.740928", + "diversity": { + "train": { + "mean_centered_corr": -0.019465479999780655, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04585251957178116 + }, + "val": { + "mean_centered_corr": -0.01962343417108059, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06466832011938095 + }, + "test": { + "mean_centered_corr": -0.016597073525190353, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08519241958856583 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..aa2a368e6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,691 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22569541421502587, + "mae": 0.11816450208425522, + "r2": 0.6745686531066895, + "score": -0.22569541421502587 + }, + "val": { + "rmse": 0.23305490182627153, + "mae": 0.1270279735326767, + "r2": 0.6371996402740479, + "score": -0.23305490182627153 + }, + "test": { + "rmse": 0.23732565171059752, + "mae": 0.15259529650211334, + "r2": 0.6151168346405029, + "score": -0.23732565171059752 + } + }, + "time": "0:01:17.740928", + "diversity": { + "train": { + "mean_centered_corr": -0.019465479999780655, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04585251957178116 + }, + "val": { + "mean_centered_corr": -0.01962343417108059, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06466832011938095 + }, + "test": { + "mean_centered_corr": -0.016597073525190353, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08519241958856583 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 25, + 21, + 29 + ], + "head_selection_time": "0:00:00.074319" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..b8e2f7b17 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2241888421744036, + "mae": 0.1289592683315277, + "r2": 0.678898811340332, + "score": -0.2241888421744036 + }, + "val": { + "rmse": 0.23918568828057105, + "mae": 0.14043618738651276, + "r2": 0.6178607940673828, + "score": -0.23918568828057105 + }, + "test": { + "rmse": 0.25293341379845147, + "mae": 0.17517992854118347, + "r2": 0.5628283023834229, + "score": -0.25293341379845147 + } + }, + "time": "0:01:18.549217", + "diversity": { + "train": { + "mean_centered_corr": -0.017244398593902588, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045789685100317 + }, + "val": { + "mean_centered_corr": -0.019243216142058372, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06452018767595291 + }, + "test": { + "mean_centered_corr": -0.017609506845474243, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08646101504564285 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21 + ], + "head_selection_time": "0:00:00.015402" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p005-evaluation/0.toml new file mode 100644 index 000000000..e8e4c7ed8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p005-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.95 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..5c0a0b3d6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p005-evaluation/0/report.json @@ -0,0 +1,684 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.220642866499973, + "mae": 0.11938578635454178, + "r2": 0.6889761686325073, + "score": -0.220642866499973 + }, + "val": { + "rmse": 0.2351252944190024, + "mae": 0.12928679585456848, + "r2": 0.6307249069213867, + "score": -0.2351252944190024 + }, + "test": { + "rmse": 0.23697534320527558, + "mae": 0.15434296429157257, + "r2": 0.6162521839141846, + "score": -0.23697534320527558 + } + }, + "time": "0:01:18.549217", + "diversity": { + "train": { + "mean_centered_corr": -0.017244398593902588, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045789685100317 + }, + "val": { + "mean_centered_corr": -0.019243216142058372, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06452018767595291 + }, + "test": { + "mean_centered_corr": -0.017609506845474243, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08646101504564285 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..f27463361 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,693 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2206414904658828, + "mae": 0.12224060297012329, + "r2": 0.6889800429344177, + "score": -0.2206414904658828 + }, + "val": { + "rmse": 0.2331196304833006, + "mae": 0.12830887734889984, + "r2": 0.6369980573654175, + "score": -0.2331196304833006 + }, + "test": { + "rmse": 0.23280496841127477, + "mae": 0.14530567824840546, + "r2": 0.6296399831771851, + "score": -0.23280496841127477 + } + }, + "time": "0:01:18.549217", + "diversity": { + "train": { + "mean_centered_corr": -0.017244398593902588, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045789685100317 + }, + "val": { + "mean_centered_corr": -0.019243216142058372, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06452018767595291 + }, + "test": { + "mean_centered_corr": -0.017609506845474243, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08646101504564285 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21, + 8, + 20, + 31, + 19, + 0 + ], + "head_selection_time": "0:00:00.102701" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..175c33ea7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2425666348719968, + "mae": 0.12444926798343658, + "r2": 0.6240968108177185, + "score": -0.2425666348719968 + }, + "val": { + "rmse": 0.23756948853650636, + "mae": 0.13087332248687744, + "r2": 0.6230076551437378, + "score": -0.23756948853650636 + }, + "test": { + "rmse": 0.24224213014777235, + "mae": 0.15561068058013916, + "r2": 0.5990049839019775, + "score": -0.24224213014777235 + } + }, + "time": "0:01:11.515736", + "diversity": { + "train": { + "mean_centered_corr": -0.019360000267624855, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04683505743741989 + }, + "val": { + "mean_centered_corr": -0.019574038684368134, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06655079871416092 + }, + "test": { + "mean_centered_corr": -0.01754024624824524, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08368789404630661 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015372" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p01-evaluation/0.toml new file mode 100644 index 000000000..859a25685 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p01-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.95 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..54e6a2233 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p01-evaluation/0/report.json @@ -0,0 +1,684 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23343200123041558, + "mae": 0.12027841061353683, + "r2": 0.6518754363059998, + "score": -0.23343200123041558 + }, + "val": { + "rmse": 0.23512438339576403, + "mae": 0.12516294419765472, + "r2": 0.6307278275489807, + "score": -0.23512438339576403 + }, + "test": { + "rmse": 0.2316782780837399, + "mae": 0.14233005046844482, + "r2": 0.633216142654419, + "score": -0.2316782780837399 + } + }, + "time": "0:01:11.515736", + "diversity": { + "train": { + "mean_centered_corr": -0.019360000267624855, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04683505743741989 + }, + "val": { + "mean_centered_corr": -0.019574038684368134, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06655079871416092 + }, + "test": { + "mean_centered_corr": -0.01754024624824524, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08368789404630661 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..905a607c5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,693 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23458808907797565, + "mae": 0.12325569242238998, + "r2": 0.648418664932251, + "score": -0.23458808907797565 + }, + "val": { + "rmse": 0.23291770554949479, + "mae": 0.1261579543352127, + "r2": 0.6376266479492188, + "score": -0.23291770554949479 + }, + "test": { + "rmse": 0.23458472246540202, + "mae": 0.14723080396652222, + "r2": 0.6239557266235352, + "score": -0.23458472246540202 + } + }, + "time": "0:01:11.515736", + "diversity": { + "train": { + "mean_centered_corr": -0.019360000267624855, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04683505743741989 + }, + "val": { + "mean_centered_corr": -0.019574038684368134, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06655079871416092 + }, + "test": { + "mean_centered_corr": -0.01754024624824524, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08368789404630661 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 25, + 21, + 6, + 19, + 0 + ], + "head_selection_time": "0:00:00.101988" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..4ce56db81 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24039030054595545, + "mae": 0.1237230896949768, + "r2": 0.6308118104934692, + "score": -0.24039030054595545 + }, + "val": { + "rmse": 0.23560357747729638, + "mae": 0.13029368221759796, + "r2": 0.6292210817337036, + "score": -0.23560357747729638 + }, + "test": { + "rmse": 0.2389191702722735, + "mae": 0.15265432000160217, + "r2": 0.6099308729171753, + "score": -0.2389191702722735 + } + }, + "time": "0:01:12.376039", + "diversity": { + "train": { + "mean_centered_corr": -0.018871087580919266, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04625045508146286 + }, + "val": { + "mean_centered_corr": -0.019568709656596184, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0614706352353096 + }, + "test": { + "mean_centered_corr": -0.018940167501568794, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08073103427886963 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015348" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/0.toml new file mode 100644 index 000000000..aff429e28 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..92245a889 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/0/report.json @@ -0,0 +1,684 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23278188472143527, + "mae": 0.12037675827741623, + "r2": 0.6538118124008179, + "score": -0.23278188472143527 + }, + "val": { + "rmse": 0.235050071622046, + "mae": 0.12535566091537476, + "r2": 0.6309611797332764, + "score": -0.235050071622046 + }, + "test": { + "rmse": 0.23184680318718037, + "mae": 0.14313000440597534, + "r2": 0.6326823234558105, + "score": -0.23184680318718037 + } + }, + "time": "0:01:12.376039", + "diversity": { + "train": { + "mean_centered_corr": -0.018871087580919266, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04625045508146286 + }, + "val": { + "mean_centered_corr": -0.019568709656596184, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0614706352353096 + }, + "test": { + "mean_centered_corr": -0.018940167501568794, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08073103427886963 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..0a73544e2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,692 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23573796221874224, + "mae": 0.12117554992437363, + "r2": 0.6449635028839111, + "score": -0.23573796221874224 + }, + "val": { + "rmse": 0.2324412121462238, + "mae": 0.12522703409194946, + "r2": 0.6391078233718872, + "score": -0.2324412121462238 + }, + "test": { + "rmse": 0.23820740696857753, + "mae": 0.1556985229253769, + "r2": 0.6122515201568604, + "score": -0.23820740696857753 + } + }, + "time": "0:01:12.376039", + "diversity": { + "train": { + "mean_centered_corr": -0.018871087580919266, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04625045508146286 + }, + "val": { + "mean_centered_corr": -0.019568709656596184, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0614706352353096 + }, + "test": { + "mean_centered_corr": -0.018940167501568794, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08073103427886963 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 0, + 1, + 25, + 21 + ], + "head_selection_time": "0:00:00.090212" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..1b85fd03e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24104706607058576, + "mae": 0.1307125687599182, + "r2": 0.6287918090820312, + "score": -0.24104706607058576 + }, + "val": { + "rmse": 0.2372450969584346, + "mae": 0.1306145042181015, + "r2": 0.624036431312561, + "score": -0.2372450969584346 + }, + "test": { + "rmse": 0.23726047684878263, + "mae": 0.14689351618289948, + "r2": 0.6153281927108765, + "score": -0.23726047684878263 + } + }, + "time": "0:01:11.064623", + "diversity": { + "train": { + "mean_centered_corr": -0.019136181101202965, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04613374546170235 + }, + "val": { + "mean_centered_corr": -0.01984967105090618, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06134215369820595 + }, + "test": { + "mean_centered_corr": -0.0187427569180727, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08017097413539886 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0 + ], + "head_selection_time": "0:00:00.015268" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p005-evaluation/0.toml new file mode 100644 index 000000000..46158c91e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p005-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..121ee2f43 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p005-evaluation/0/report.json @@ -0,0 +1,684 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23274665867971067, + "mae": 0.1198442280292511, + "r2": 0.653916597366333, + "score": -0.23274665867971067 + }, + "val": { + "rmse": 0.23505585640746507, + "mae": 0.12548410892486572, + "r2": 0.6309430599212646, + "score": -0.23505585640746507 + }, + "test": { + "rmse": 0.23276235981389973, + "mae": 0.14522086083889008, + "r2": 0.6297755241394043, + "score": -0.23276235981389973 + } + }, + "time": "0:01:11.064623", + "diversity": { + "train": { + "mean_centered_corr": -0.019136181101202965, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04613374546170235 + }, + "val": { + "mean_centered_corr": -0.01984967105090618, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06134215369820595 + }, + "test": { + "mean_centered_corr": -0.0187427569180727, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08017097413539886 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..7aa242102 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,696 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23595696457409665, + "mae": 0.12094593048095703, + "r2": 0.644303560256958, + "score": -0.23595696457409665 + }, + "val": { + "rmse": 0.2328726380397863, + "mae": 0.12572719156742096, + "r2": 0.6377668380737305, + "score": -0.2328726380397863 + }, + "test": { + "rmse": 0.23808236498886684, + "mae": 0.15540435910224915, + "r2": 0.6126585006713867, + "score": -0.23808236498886684 + } + }, + "time": "0:01:11.064623", + "diversity": { + "train": { + "mean_centered_corr": -0.019136181101202965, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04613374546170235 + }, + "val": { + "mean_centered_corr": -0.01984967105090618, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06134215369820595 + }, + "test": { + "mean_centered_corr": -0.0187427569180727, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08017097413539886 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0, + 4, + 16, + 1, + 10, + 25, + 21, + 16, + 4 + ], + "head_selection_time": "0:00:00.145331" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..46f33a2a2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24078892072406902, + "mae": 0.12140180170536041, + "r2": 0.6295864582061768, + "score": -0.24078892072406902 + }, + "val": { + "rmse": 0.23656618722639788, + "mae": 0.1301935315132141, + "r2": 0.6261851191520691, + "score": -0.23656618722639788 + }, + "test": { + "rmse": 0.2443526301182123, + "mae": 0.15885169804096222, + "r2": 0.5919873714447021, + "score": -0.2443526301182123 + } + }, + "time": "0:01:10.820742", + "diversity": { + "train": { + "mean_centered_corr": -0.019902383908629417, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04721210524439812 + }, + "val": { + "mean_centered_corr": -0.019949622452259064, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06361114978790283 + }, + "test": { + "mean_centered_corr": -0.019116314128041267, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07747960835695267 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015547" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p01-evaluation/0.toml new file mode 100644 index 000000000..4e30093ec --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p01-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..ac383e963 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p01-evaluation/0/report.json @@ -0,0 +1,684 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23225320578180678, + "mae": 0.12026127427816391, + "r2": 0.655382513999939, + "score": -0.23225320578180678 + }, + "val": { + "rmse": 0.23504784483792887, + "mae": 0.12635678052902222, + "r2": 0.6309682130813599, + "score": -0.23504784483792887 + }, + "test": { + "rmse": 0.23465037063607563, + "mae": 0.14887936413288116, + "r2": 0.6237452030181885, + "score": -0.23465037063607563 + } + }, + "time": "0:01:10.820742", + "diversity": { + "train": { + "mean_centered_corr": -0.019902383908629417, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04721210524439812 + }, + "val": { + "mean_centered_corr": -0.019949622452259064, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06361114978790283 + }, + "test": { + "mean_centered_corr": -0.019116314128041267, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07747960835695267 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..64930bc5e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,695 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2344874747922904, + "mae": 0.1211647242307663, + "r2": 0.6487202048301697, + "score": -0.2344874747922904 + }, + "val": { + "rmse": 0.23334387616340208, + "mae": 0.12681756913661957, + "r2": 0.6362993717193604, + "score": -0.23334387616340208 + }, + "test": { + "rmse": 0.2380340654227449, + "mae": 0.15519878268241882, + "r2": 0.6128156185150146, + "score": -0.2380340654227449 + } + }, + "time": "0:01:10.820742", + "diversity": { + "train": { + "mean_centered_corr": -0.019902383908629417, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04721210524439812 + }, + "val": { + "mean_centered_corr": -0.019949622452259064, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06361114978790283 + }, + "test": { + "mean_centered_corr": -0.019116314128041267, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07747960835695267 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 29, + 19, + 1, + 10, + 25, + 21, + 16 + ], + "head_selection_time": "0:00:00.131017" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..1db8be0ad --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24066084712400115, + "mae": 0.1302179992198944, + "r2": 0.6299803256988525, + "score": -0.24066084712400115 + }, + "val": { + "rmse": 0.23897181954001903, + "mae": 0.13393160700798035, + "r2": 0.6185438632965088, + "score": -0.23897181954001903 + }, + "test": { + "rmse": 0.23496733361066027, + "mae": 0.1394679695367813, + "r2": 0.6227279901504517, + "score": -0.23496733361066027 + } + }, + "time": "0:01:10.329042", + "diversity": { + "train": { + "mean_centered_corr": -0.01913526840507984, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04782585799694061 + }, + "val": { + "mean_centered_corr": -0.01699827052652836, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06600741297006607 + }, + "test": { + "mean_centered_corr": -0.020212531089782715, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08567789196968079 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 8 + ], + "head_selection_time": "0:00:00.015400" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p001-evaluation/0.toml new file mode 100644 index 000000000..bcd7aae35 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p001-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..3649d701d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p001-evaluation/0/report.json @@ -0,0 +1,684 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2318619145508088, + "mae": 0.12150537222623825, + "r2": 0.6565427184104919, + "score": -0.2318619145508088 + }, + "val": { + "rmse": 0.2357328737026148, + "mae": 0.12986169755458832, + "r2": 0.6288140416145325, + "score": -0.2357328737026148 + }, + "test": { + "rmse": 0.2351772485906963, + "mae": 0.14941059052944183, + "r2": 0.6220536231994629, + "score": -0.2351772485906963 + } + }, + "time": "0:01:10.329042", + "diversity": { + "train": { + "mean_centered_corr": -0.01913526840507984, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04782585799694061 + }, + "val": { + "mean_centered_corr": -0.01699827052652836, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06600741297006607 + }, + "test": { + "mean_centered_corr": -0.020212531089782715, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08567789196968079 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..1ceba83b0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,699 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23260143300346486, + "mae": 0.12271209806203842, + "r2": 0.6543483734130859, + "score": -0.23260143300346486 + }, + "val": { + "rmse": 0.2341784288692296, + "mae": 0.1288924217224121, + "r2": 0.6336931586265564, + "score": -0.2341784288692296 + }, + "test": { + "rmse": 0.23454983071534766, + "mae": 0.14984872937202454, + "r2": 0.6240675449371338, + "score": -0.23454983071534766 + } + }, + "time": "0:01:10.329042", + "diversity": { + "train": { + "mean_centered_corr": -0.01913526840507984, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04782585799694061 + }, + "val": { + "mean_centered_corr": -0.01699827052652836, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06600741297006607 + }, + "test": { + "mean_centered_corr": -0.020212531089782715, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08567789196968079 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 8, + 21, + 27, + 19, + 3, + 25, + 4, + 12, + 0, + 18, + 3, + 24 + ], + "head_selection_time": "0:00:00.196120" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..44a38d101 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1184, + "metrics": { + "train": { + "rmse": 0.22160140516469537, + "mae": 0.12272227555513382, + "r2": 0.6862679719924927, + "score": -0.22160140516469537 + }, + "val": { + "rmse": 0.24058462987099405, + "mae": 0.13246983289718628, + "r2": 0.613377571105957, + "score": -0.24058462987099405 + }, + "test": { + "rmse": 0.2859647081281448, + "mae": 0.2032262533903122, + "r2": 0.44118958711624146, + "score": -0.2859647081281448 + } + }, + "time": "0:01:30.096626", + "diversity": { + "train": { + "mean_centered_corr": -0.0177434291690588, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05118265748023987 + }, + "val": { + "mean_centered_corr": -0.01485136803239584, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0833396464586258 + }, + "test": { + "mean_centered_corr": -0.015522093512117863, + "mean_pairwise_disagreement": NaN, + "member_std": 0.11537164449691772 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 10 + ], + "head_selection_time": "0:00:00.015160" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p005-evaluation/0.toml new file mode 100644 index 000000000..2724d4773 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p005-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..85c64d585 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p005-evaluation/0/report.json @@ -0,0 +1,684 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1184, + "metrics": { + "train": { + "rmse": 0.2047007403568582, + "mae": 0.11088933050632477, + "r2": 0.7322973012924194, + "score": -0.2047007403568582 + }, + "val": { + "rmse": 0.2351519581095747, + "mae": 0.12822481989860535, + "r2": 0.6306412220001221, + "score": -0.2351519581095747 + }, + "test": { + "rmse": 0.2408751648791966, + "mae": 0.15764909982681274, + "r2": 0.6035178899765015, + "score": -0.2408751648791966 + } + }, + "time": "0:01:30.096626", + "diversity": { + "train": { + "mean_centered_corr": -0.0177434291690588, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05118265748023987 + }, + "val": { + "mean_centered_corr": -0.01485136803239584, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0833396464586258 + }, + "test": { + "mean_centered_corr": -0.015522093512117863, + "mean_pairwise_disagreement": NaN, + "member_std": 0.11537164449691772 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..4f904616f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,705 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1184, + "metrics": { + "train": { + "rmse": 0.20677708372923248, + "mae": 0.11196121573448181, + "r2": 0.7268389463424683, + "score": -0.20677708372923248 + }, + "val": { + "rmse": 0.23316436266516308, + "mae": 0.12652936577796936, + "r2": 0.6368587017059326, + "score": -0.23316436266516308 + }, + "test": { + "rmse": 0.23171057167720172, + "mae": 0.14218251407146454, + "r2": 0.6331138610839844, + "score": -0.23171057167720172 + } + }, + "time": "0:01:30.096626", + "diversity": { + "train": { + "mean_centered_corr": -0.0177434291690588, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05118265748023987 + }, + "val": { + "mean_centered_corr": -0.01485136803239584, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0833396464586258 + }, + "test": { + "mean_centered_corr": -0.015522093512117863, + "mean_pairwise_disagreement": NaN, + "member_std": 0.11537164449691772 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 10, + 27, + 7, + 1, + 23, + 16, + 29, + 5, + 17, + 7, + 25, + 29, + 20, + 27, + 1, + 21, + 7, + 22 + ], + "head_selection_time": "0:00:00.279741" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..0e1f8f255 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2438513529798656, + "mae": 0.12642574310302734, + "r2": 0.6201044321060181, + "score": -0.2438513529798656 + }, + "val": { + "rmse": 0.23781902006036262, + "mae": 0.13461825251579285, + "r2": 0.6222152709960938, + "score": -0.23781902006036262 + }, + "test": { + "rmse": 0.2470883298795412, + "mae": 0.16668537259101868, + "r2": 0.5828002095222473, + "score": -0.2470883298795412 + } + }, + "time": "0:01:11.258846", + "diversity": { + "train": { + "mean_centered_corr": -0.01934780180454254, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04824277386069298 + }, + "val": { + "mean_centered_corr": -0.018993837758898735, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06548074632883072 + }, + "test": { + "mean_centered_corr": -0.019981293007731438, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0883246660232544 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015208" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p01-evaluation/0.toml new file mode 100644 index 000000000..65c4d5ef3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p01-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..32c548ccb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p01-evaluation/0/report.json @@ -0,0 +1,684 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23279507911071862, + "mae": 0.12109043449163437, + "r2": 0.6537725925445557, + "score": -0.23279507911071862 + }, + "val": { + "rmse": 0.2353263382433216, + "mae": 0.12844589352607727, + "r2": 0.6300932168960571, + "score": -0.2353263382433216 + }, + "test": { + "rmse": 0.2351845825591179, + "mae": 0.1493937373161316, + "r2": 0.6220300197601318, + "score": -0.2351845825591179 + } + }, + "time": "0:01:11.258846", + "diversity": { + "train": { + "mean_centered_corr": -0.01934780180454254, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04824277386069298 + }, + "val": { + "mean_centered_corr": -0.018993837758898735, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06548074632883072 + }, + "test": { + "mean_centered_corr": -0.019981293007731438, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0883246660232544 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..2d7b9734f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,698 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23261467766548827, + "mae": 0.12115577608346939, + "r2": 0.6543089747428894, + "score": -0.23261467766548827 + }, + "val": { + "rmse": 0.2335463179080623, + "mae": 0.127370223402977, + "r2": 0.6356680393218994, + "score": -0.2335463179080623 + }, + "test": { + "rmse": 0.23522276143697368, + "mae": 0.15016508102416992, + "r2": 0.6219072937965393, + "score": -0.23522276143697368 + } + }, + "time": "0:01:11.258846", + "diversity": { + "train": { + "mean_centered_corr": -0.01934780180454254, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04824277386069298 + }, + "val": { + "mean_centered_corr": -0.018993837758898735, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06548074632883072 + }, + "test": { + "mean_centered_corr": -0.019981293007731438, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0883246660232544 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 19, + 1, + 20, + 25, + 12, + 21, + 25, + 3, + 28, + 19 + ], + "head_selection_time": "0:00:00.174015" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..b4357867a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24166333481256672, + "mae": 0.1247403547167778, + "r2": 0.6268912553787231, + "score": -0.24166333481256672 + }, + "val": { + "rmse": 0.23872630190199182, + "mae": 0.12959358096122742, + "r2": 0.6193272471427917, + "score": -0.23872630190199182 + }, + "test": { + "rmse": 0.23512648270487807, + "mae": 0.1454058140516281, + "r2": 0.6222167611122131, + "score": -0.23512648270487807 + } + }, + "time": "0:01:32.555768", + "diversity": { + "train": { + "mean_centered_corr": -0.01866377331316471, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04515796899795532 + }, + "val": { + "mean_centered_corr": -0.019534021615982056, + "mean_pairwise_disagreement": NaN, + "member_std": 0.059918854385614395 + }, + "test": { + "mean_centered_corr": -0.016633860766887665, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0777917131781578 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015199" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p001-evaluation/0.toml new file mode 100644 index 000000000..efb5a9c0b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p001-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.95 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..5024a184c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p001-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23281516131217675, + "mae": 0.12193623930215836, + "r2": 0.653712809085846, + "score": -0.23281516131217675 + }, + "val": { + "rmse": 0.23518726739965692, + "mae": 0.12629030644893646, + "r2": 0.6305302381515503, + "score": -0.23518726739965692 + }, + "test": { + "rmse": 0.23268638913916373, + "mae": 0.14553509652614594, + "r2": 0.6300171613693237, + "score": -0.23268638913916373 + } + }, + "time": "0:01:32.555768", + "diversity": { + "train": { + "mean_centered_corr": -0.01866377331316471, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04515796899795532 + }, + "val": { + "mean_centered_corr": -0.019534021615982056, + "mean_pairwise_disagreement": NaN, + "member_std": 0.059918854385614395 + }, + "test": { + "mean_centered_corr": -0.016633860766887665, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0777917131781578 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..ec2e59f69 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,700 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2349319119810031, + "mae": 0.12289345264434814, + "r2": 0.6473873257637024, + "score": -0.2349319119810031 + }, + "val": { + "rmse": 0.23347937804489072, + "mae": 0.12790052592754364, + "r2": 0.6358768343925476, + "score": -0.23347937804489072 + }, + "test": { + "rmse": 0.24206068964408525, + "mae": 0.16086436808109283, + "r2": 0.5996055006980896, + "score": -0.24206068964408525 + } + }, + "time": "0:01:32.555768", + "diversity": { + "train": { + "mean_centered_corr": -0.01866377331316471, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04515796899795532 + }, + "val": { + "mean_centered_corr": -0.019534021615982056, + "mean_pairwise_disagreement": NaN, + "member_std": 0.059918854385614395 + }, + "test": { + "mean_centered_corr": -0.016633860766887665, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0777917131781578 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 25, + 21, + 1, + 19, + 25, + 10 + ], + "head_selection_time": "0:00:00.116290" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..a0a0afcdb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24294329966861838, + "mae": 0.12473063915967941, + "r2": 0.6229285001754761, + "score": -0.24294329966861838 + }, + "val": { + "rmse": 0.23800428115225924, + "mae": 0.13076891005039215, + "r2": 0.6216264367103577, + "score": -0.23800428115225924 + }, + "test": { + "rmse": 0.23509819987146183, + "mae": 0.1425461322069168, + "r2": 0.6223076581954956, + "score": -0.23509819987146183 + } + }, + "time": "0:01:32.973453", + "diversity": { + "train": { + "mean_centered_corr": -0.01892852410674095, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046223822981119156 + }, + "val": { + "mean_centered_corr": -0.019010689109563828, + "mean_pairwise_disagreement": NaN, + "member_std": 0.061593979597091675 + }, + "test": { + "mean_centered_corr": -0.01590883731842041, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07384058833122253 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015239" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p005-evaluation/0.toml new file mode 100644 index 000000000..4c913eb52 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p005-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.95 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..193effd39 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p005-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23234710773024822, + "mae": 0.12129862606525421, + "r2": 0.6551038026809692, + "score": -0.23234710773024822 + }, + "val": { + "rmse": 0.23483364211643792, + "mae": 0.12628823518753052, + "r2": 0.6316404938697815, + "score": -0.23483364211643792 + }, + "test": { + "rmse": 0.23156642561711185, + "mae": 0.14394673705101013, + "r2": 0.6335701942443848, + "score": -0.23156642561711185 + } + }, + "time": "0:01:32.973453", + "diversity": { + "train": { + "mean_centered_corr": -0.01892852410674095, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046223822981119156 + }, + "val": { + "mean_centered_corr": -0.019010689109563828, + "mean_pairwise_disagreement": NaN, + "member_std": 0.061593979597091675 + }, + "test": { + "mean_centered_corr": -0.01590883731842041, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07384058833122253 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..9b58999fc --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,702 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23416801690807423, + "mae": 0.12230122834444046, + "r2": 0.6496766805648804, + "score": -0.23416801690807423 + }, + "val": { + "rmse": 0.23277299468998114, + "mae": 0.12583908438682556, + "r2": 0.6380767822265625, + "score": -0.23277299468998114 + }, + "test": { + "rmse": 0.2357753957826797, + "mae": 0.15136900544166565, + "r2": 0.6201286315917969, + "score": -0.2357753957826797 + } + }, + "time": "0:01:32.973453", + "diversity": { + "train": { + "mean_centered_corr": -0.01892852410674095, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046223822981119156 + }, + "val": { + "mean_centered_corr": -0.019010689109563828, + "mean_pairwise_disagreement": NaN, + "member_std": 0.061593979597091675 + }, + "test": { + "mean_centered_corr": -0.01590883731842041, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07384058833122253 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 6, + 0, + 19, + 1, + 25, + 21, + 25, + 1 + ], + "head_selection_time": "0:00:00.144542" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..b425a37ad --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.21889456161617807, + "mae": 0.12161961197853088, + "r2": 0.6938855648040771, + "score": -0.21889456161617807 + }, + "val": { + "rmse": 0.23798911368843165, + "mae": 0.13175897300243378, + "r2": 0.621674656867981, + "score": -0.23798911368843165 + }, + "test": { + "rmse": 0.24638280846527752, + "mae": 0.16409903764724731, + "r2": 0.585179328918457, + "score": -0.24638280846527752 + } + }, + "time": "0:01:41.397939", + "diversity": { + "train": { + "mean_centered_corr": -0.01843600906431675, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046984437853097916 + }, + "val": { + "mean_centered_corr": -0.01927182450890541, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06631730496883392 + }, + "test": { + "mean_centered_corr": -0.018402421846985817, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08607663214206696 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21 + ], + "head_selection_time": "0:00:00.015360" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p01-evaluation/0.toml new file mode 100644 index 000000000..b700865b4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p01-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.95 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..96e694441 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p01-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2200473901522814, + "mae": 0.11505363136529922, + "r2": 0.6906527280807495, + "score": -0.2200473901522814 + }, + "val": { + "rmse": 0.23457506700663386, + "mae": 0.12505945563316345, + "r2": 0.6324512362480164, + "score": -0.23457506700663386 + }, + "test": { + "rmse": 0.2318295295556057, + "mae": 0.1426994651556015, + "r2": 0.6327370405197144, + "score": -0.2318295295556057 + } + }, + "time": "0:01:41.397939", + "diversity": { + "train": { + "mean_centered_corr": -0.01843600906431675, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046984437853097916 + }, + "val": { + "mean_centered_corr": -0.01927182450890541, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06631730496883392 + }, + "test": { + "mean_centered_corr": -0.018402421846985817, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08607663214206696 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..580711c62 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,697 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.22139641227752963, + "mae": 0.12048649042844772, + "r2": 0.6868481040000916, + "score": -0.22139641227752963 + }, + "val": { + "rmse": 0.2329412795410917, + "mae": 0.12749527394771576, + "r2": 0.637553334236145, + "score": -0.2329412795410917 + }, + "test": { + "rmse": 0.23349850795747007, + "mae": 0.14457938075065613, + "r2": 0.6274300813674927, + "score": -0.23349850795747007 + } + }, + "time": "0:01:41.397939", + "diversity": { + "train": { + "mean_centered_corr": -0.01843600906431675, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046984437853097916 + }, + "val": { + "mean_centered_corr": -0.01927182450890541, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06631730496883392 + }, + "test": { + "mean_centered_corr": -0.018402421846985817, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08607663214206696 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21, + 8, + 31, + 0 + ], + "head_selection_time": "0:00:00.074215" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..83b9d9524 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2248883132434097, + "mae": 0.12571877241134644, + "r2": 0.6768920421600342, + "score": -0.2248883132434097 + }, + "val": { + "rmse": 0.23993622491293543, + "mae": 0.13967159390449524, + "r2": 0.6154587864875793, + "score": -0.23993622491293543 + }, + "test": { + "rmse": 0.2477713353601464, + "mae": 0.16416500508785248, + "r2": 0.5804905891418457, + "score": -0.2477713353601464 + } + }, + "time": "0:01:42.023156", + "diversity": { + "train": { + "mean_centered_corr": -0.017943480983376503, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04725648835301399 + }, + "val": { + "mean_centered_corr": -0.017995651811361313, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07173214852809906 + }, + "test": { + "mean_centered_corr": -0.018061438575387, + "mean_pairwise_disagreement": NaN, + "member_std": 0.10140956938266754 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015426" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p001-evaluation/0.toml new file mode 100644 index 000000000..46394825d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p001-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..4f69b08da --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p001-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.217070278259022, + "mae": 0.11539068818092346, + "r2": 0.6989666223526001, + "score": -0.217070278259022 + }, + "val": { + "rmse": 0.2351439260504182, + "mae": 0.1256217062473297, + "r2": 0.6306663751602173, + "score": -0.2351439260504182 + }, + "test": { + "rmse": 0.23458613581335708, + "mae": 0.1473771333694458, + "r2": 0.6239511370658875, + "score": -0.23458613581335708 + } + }, + "time": "0:01:42.023156", + "diversity": { + "train": { + "mean_centered_corr": -0.017943480983376503, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04725648835301399 + }, + "val": { + "mean_centered_corr": -0.017995651811361313, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07173214852809906 + }, + "test": { + "mean_centered_corr": -0.018061438575387, + "mean_pairwise_disagreement": NaN, + "member_std": 0.10140956938266754 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..dc63bb828 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,698 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.22117446261892904, + "mae": 0.11840678006410599, + "r2": 0.6874756813049316, + "score": -0.22117446261892904 + }, + "val": { + "rmse": 0.23356126348868944, + "mae": 0.12733577191829681, + "r2": 0.6356213688850403, + "score": -0.23356126348868944 + }, + "test": { + "rmse": 0.2502618550799444, + "mae": 0.168344646692276, + "r2": 0.5720146894454956, + "score": -0.2502618550799444 + } + }, + "time": "0:01:42.023156", + "diversity": { + "train": { + "mean_centered_corr": -0.017943480983376503, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04725648835301399 + }, + "val": { + "mean_centered_corr": -0.017995651811361313, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07173214852809906 + }, + "test": { + "mean_centered_corr": -0.018061438575387, + "mean_pairwise_disagreement": NaN, + "member_std": 0.10140956938266754 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 25, + 0, + 19, + 10 + ], + "head_selection_time": "0:00:00.088092" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..4238dea50 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24031171096652382, + "mae": 0.12308599799871445, + "r2": 0.6310532093048096, + "score": -0.24031171096652382 + }, + "val": { + "rmse": 0.2360115531890879, + "mae": 0.13377444446086884, + "r2": 0.6279358863830566, + "score": -0.2360115531890879 + }, + "test": { + "rmse": 0.23473422019733492, + "mae": 0.14439672231674194, + "r2": 0.6234762668609619, + "score": -0.23473422019733492 + } + }, + "time": "0:01:30.869624", + "diversity": { + "train": { + "mean_centered_corr": -0.01947913132607937, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0460021011531353 + }, + "val": { + "mean_centered_corr": -0.0197152066975832, + "mean_pairwise_disagreement": NaN, + "member_std": 0.061192721128463745 + }, + "test": { + "mean_centered_corr": -0.017801720649003983, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07759672403335571 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015163" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p005-evaluation/0.toml new file mode 100644 index 000000000..ad5a60008 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p005-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..108a0a152 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p005-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2315330821822192, + "mae": 0.11987455189228058, + "r2": 0.6575162410736084, + "score": -0.2315330821822192 + }, + "val": { + "rmse": 0.2346577290091983, + "mae": 0.12638406455516815, + "r2": 0.6321921348571777, + "score": -0.2346577290091983 + }, + "test": { + "rmse": 0.23333821657015275, + "mae": 0.1463044285774231, + "r2": 0.6279414296150208, + "score": -0.23333821657015275 + } + }, + "time": "0:01:30.869624", + "diversity": { + "train": { + "mean_centered_corr": -0.01947913132607937, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0460021011531353 + }, + "val": { + "mean_centered_corr": -0.0197152066975832, + "mean_pairwise_disagreement": NaN, + "member_std": 0.061192721128463745 + }, + "test": { + "mean_centered_corr": -0.017801720649003983, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07759672403335571 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..0969748b8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,699 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23331299019728063, + "mae": 0.12291000783443451, + "r2": 0.6522302627563477, + "score": -0.23331299019728063 + }, + "val": { + "rmse": 0.23283208982244355, + "mae": 0.12645916640758514, + "r2": 0.6378929615020752, + "score": -0.23283208982244355 + }, + "test": { + "rmse": 0.23176779995051228, + "mae": 0.1443839818239212, + "r2": 0.6329326629638672, + "score": -0.23176779995051228 + } + }, + "time": "0:01:30.869624", + "diversity": { + "train": { + "mean_centered_corr": -0.01947913132607937, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0460021011531353 + }, + "val": { + "mean_centered_corr": -0.0197152066975832, + "mean_pairwise_disagreement": NaN, + "member_std": 0.061192721128463745 + }, + "test": { + "mean_centered_corr": -0.017801720649003983, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07759672403335571 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 16, + 0, + 21, + 1, + 19 + ], + "head_selection_time": "0:00:00.101259" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..3a53f5dea --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2411780624535083, + "mae": 0.12449400126934052, + "r2": 0.6283881664276123, + "score": -0.2411780624535083 + }, + "val": { + "rmse": 0.23637143045682502, + "mae": 0.13214340806007385, + "r2": 0.6268003582954407, + "score": -0.23637143045682502 + }, + "test": { + "rmse": 0.2444339973181191, + "mae": 0.1604158580303192, + "r2": 0.5917155742645264, + "score": -0.2444339973181191 + } + }, + "time": "0:01:30.058806", + "diversity": { + "train": { + "mean_centered_corr": -0.01962215080857277, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04651423916220665 + }, + "val": { + "mean_centered_corr": -0.019082706421613693, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06017685681581497 + }, + "test": { + "mean_centered_corr": -0.01942536234855652, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08030469715595245 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015428" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p01-evaluation/0.toml new file mode 100644 index 000000000..eca59c06e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p01-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..eea0cd4ec --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p01-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23221441033396834, + "mae": 0.12085375189781189, + "r2": 0.6554976105690002, + "score": -0.23221441033396834 + }, + "val": { + "rmse": 0.23515957801466283, + "mae": 0.1254815012216568, + "r2": 0.6306172609329224, + "score": -0.23515957801466283 + }, + "test": { + "rmse": 0.23255229143999848, + "mae": 0.14425288140773773, + "r2": 0.6304435133934021, + "score": -0.23255229143999848 + } + }, + "time": "0:01:30.058806", + "diversity": { + "train": { + "mean_centered_corr": -0.01962215080857277, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04651423916220665 + }, + "val": { + "mean_centered_corr": -0.019082706421613693, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06017685681581497 + }, + "test": { + "mean_centered_corr": -0.01942536234855652, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08030469715595245 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..9432dce69 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,701 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23465978486724487, + "mae": 0.12119866162538528, + "r2": 0.6482037305831909, + "score": -0.23465978486724487 + }, + "val": { + "rmse": 0.233185579288438, + "mae": 0.12584175169467926, + "r2": 0.6367926597595215, + "score": -0.233185579288438 + }, + "test": { + "rmse": 0.23514442509167285, + "mae": 0.15004310011863708, + "r2": 0.6221591234207153, + "score": -0.23514442509167285 + } + }, + "time": "0:01:30.058806", + "diversity": { + "train": { + "mean_centered_corr": -0.01962215080857277, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04651423916220665 + }, + "val": { + "mean_centered_corr": -0.019082706421613693, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06017685681581497 + }, + "test": { + "mean_centered_corr": -0.01942536234855652, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08030469715595245 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 0, + 25, + 1, + 28, + 16, + 21, + 6 + ], + "head_selection_time": "0:00:00.131699" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..6deb7eeed --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24152145784724474, + "mae": 0.12505307793617249, + "r2": 0.6273292303085327, + "score": -0.24152145784724474 + }, + "val": { + "rmse": 0.2382472279758646, + "mae": 0.13743972778320312, + "r2": 0.6208535432815552, + "score": -0.2382472279758646 + }, + "test": { + "rmse": 0.2416035703731357, + "mae": 0.15706877410411835, + "r2": 0.6011162996292114, + "score": -0.2416035703731357 + } + }, + "time": "0:01:33.524887", + "diversity": { + "train": { + "mean_centered_corr": -0.018592942506074905, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046543560922145844 + }, + "val": { + "mean_centered_corr": -0.01946503296494484, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06134062260389328 + }, + "test": { + "mean_centered_corr": -0.01856323331594467, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08496059477329254 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015238" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p001-evaluation/0.toml new file mode 100644 index 000000000..091455f42 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p001-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..9cef9a45b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p001-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2314695997411829, + "mae": 0.12160827219486237, + "r2": 0.6577039957046509, + "score": -0.2314695997411829 + }, + "val": { + "rmse": 0.2349283758671495, + "mae": 0.12933029234409332, + "r2": 0.6313432455062866, + "score": -0.2349283758671495 + }, + "test": { + "rmse": 0.23427005643589985, + "mae": 0.1490129977464676, + "r2": 0.6249638199806213, + "score": -0.23427005643589985 + } + }, + "time": "0:01:33.524887", + "diversity": { + "train": { + "mean_centered_corr": -0.018592942506074905, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046543560922145844 + }, + "val": { + "mean_centered_corr": -0.01946503296494484, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06134062260389328 + }, + "test": { + "mean_centered_corr": -0.01856323331594467, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08496059477329254 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..72a253d60 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,703 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2324655315526619, + "mae": 0.1217086911201477, + "r2": 0.6547521352767944, + "score": -0.2324655315526619 + }, + "val": { + "rmse": 0.23350649291605036, + "mae": 0.12837612628936768, + "r2": 0.6357922554016113, + "score": -0.23350649291605036 + }, + "test": { + "rmse": 0.23477688348213874, + "mae": 0.14960812032222748, + "r2": 0.6233393549919128, + "score": -0.23477688348213874 + } + }, + "time": "0:01:33.524887", + "diversity": { + "train": { + "mean_centered_corr": -0.018592942506074905, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046543560922145844 + }, + "val": { + "mean_centered_corr": -0.01946503296494484, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06134062260389328 + }, + "test": { + "mean_centered_corr": -0.01856323331594467, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08496059477329254 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 19, + 20, + 3, + 29, + 25, + 12, + 27, + 25, + 10 + ], + "head_selection_time": "0:00:00.158843" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..170597da7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24317591994456592, + "mae": 0.12499266862869263, + "r2": 0.6222060322761536, + "score": -0.24317591994456592 + }, + "val": { + "rmse": 0.23764755063810214, + "mae": 0.13410833477973938, + "r2": 0.6227598190307617, + "score": -0.23764755063810214 + }, + "test": { + "rmse": 0.24545806951373206, + "mae": 0.16299617290496826, + "r2": 0.588287353515625, + "score": -0.24545806951373206 + } + }, + "time": "0:01:30.594432", + "diversity": { + "train": { + "mean_centered_corr": -0.01936914771795273, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0473526269197464 + }, + "val": { + "mean_centered_corr": -0.019165920093655586, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06515220552682877 + }, + "test": { + "mean_centered_corr": -0.020623264834284782, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08508463948965073 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015365" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p005-evaluation/0.toml new file mode 100644 index 000000000..0b70f916d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p005-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..7d43cf48c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p005-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23234297110267305, + "mae": 0.12101900577545166, + "r2": 0.655116081237793, + "score": -0.23234297110267305 + }, + "val": { + "rmse": 0.23481945176098018, + "mae": 0.12734994292259216, + "r2": 0.6316850185394287, + "score": -0.23481945176098018 + }, + "test": { + "rmse": 0.23330914213736356, + "mae": 0.1461166888475418, + "r2": 0.6280341148376465, + "score": -0.23330914213736356 + } + }, + "time": "0:01:30.594432", + "diversity": { + "train": { + "mean_centered_corr": -0.01936914771795273, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0473526269197464 + }, + "val": { + "mean_centered_corr": -0.019165920093655586, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06515220552682877 + }, + "test": { + "mean_centered_corr": -0.020623264834284782, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08508463948965073 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..62ec5b8fc --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,698 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23521274413986984, + "mae": 0.12193334102630615, + "r2": 0.6465438008308411, + "score": -0.23521274413986984 + }, + "val": { + "rmse": 0.23351058500178656, + "mae": 0.12763343751430511, + "r2": 0.6357795000076294, + "score": -0.23351058500178656 + }, + "test": { + "rmse": 0.23619933599335108, + "mae": 0.15221840143203735, + "r2": 0.6187613010406494, + "score": -0.23619933599335108 + } + }, + "time": "0:01:30.594432", + "diversity": { + "train": { + "mean_centered_corr": -0.01936914771795273, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0473526269197464 + }, + "val": { + "mean_centered_corr": -0.019165920093655586, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06515220552682877 + }, + "test": { + "mean_centered_corr": -0.020623264834284782, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08508463948965073 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 19, + 10, + 25, + 3 + ], + "head_selection_time": "0:00:00.087579" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..7a0c87fb6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.23319688178416265, + "mae": 0.12308622896671295, + "r2": 0.6525763273239136, + "score": -0.23319688178416265 + }, + "val": { + "rmse": 0.2388627040189618, + "mae": 0.13744254410266876, + "r2": 0.6188920736312866, + "score": -0.2388627040189618 + }, + "test": { + "rmse": 0.24858429278757524, + "mae": 0.16933724284172058, + "r2": 0.5777331590652466, + "score": -0.24858429278757524 + } + }, + "time": "0:01:40.878188", + "diversity": { + "train": { + "mean_centered_corr": -0.01871233992278576, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04815390333533287 + }, + "val": { + "mean_centered_corr": -0.01693091168999672, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06701929122209549 + }, + "test": { + "mean_centered_corr": -0.018649011850357056, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09262414276599884 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25 + ], + "head_selection_time": "0:00:00.015240" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p01-evaluation/0.toml new file mode 100644 index 000000000..0567ba4d7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p01-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..b9a89f1af --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p01-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.22028603428626464, + "mae": 0.11624135076999664, + "r2": 0.6899813413619995, + "score": -0.22028603428626464 + }, + "val": { + "rmse": 0.235230434477205, + "mae": 0.12826579809188843, + "r2": 0.6303946375846863, + "score": -0.235230434477205 + }, + "test": { + "rmse": 0.23369840873687428, + "mae": 0.14666512608528137, + "r2": 0.6267918348312378, + "score": -0.23369840873687428 + } + }, + "time": "0:01:40.878188", + "diversity": { + "train": { + "mean_centered_corr": -0.01871233992278576, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04815390333533287 + }, + "val": { + "mean_centered_corr": -0.01693091168999672, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06701929122209549 + }, + "test": { + "mean_centered_corr": -0.018649011850357056, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09262414276599884 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..a37b48623 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,708 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.22111136733588857, + "mae": 0.11676743626594543, + "r2": 0.6876538991928101, + "score": -0.22111136733588857 + }, + "val": { + "rmse": 0.23403326393093224, + "mae": 0.12714654207229614, + "r2": 0.6341471672058105, + "score": -0.23403326393093224 + }, + "test": { + "rmse": 0.23267639873508839, + "mae": 0.1456754505634308, + "r2": 0.630048930644989, + "score": -0.23267639873508839 + } + }, + "time": "0:01:40.878188", + "diversity": { + "train": { + "mean_centered_corr": -0.01871233992278576, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04815390333533287 + }, + "val": { + "mean_centered_corr": -0.01693091168999672, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06701929122209549 + }, + "test": { + "mean_centered_corr": -0.018649011850357056, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09262414276599884 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25, + 19, + 10, + 23, + 26, + 12, + 22, + 17, + 20, + 28, + 8, + 21, + 23, + 22, + 19 + ], + "head_selection_time": "0:00:00.235823" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..180ceb2b4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23904784974230414, + "mae": 0.12625300884246826, + "r2": 0.6349237561225891, + "score": -0.23904784974230414 + }, + "val": { + "rmse": 0.23805604517030857, + "mae": 0.13564030826091766, + "r2": 0.621461808681488, + "score": -0.23805604517030857 + }, + "test": { + "rmse": 0.24432422600723777, + "mae": 0.15747275948524475, + "r2": 0.5920822024345398, + "score": -0.24432422600723777 + } + }, + "time": "0:01:31.232910", + "diversity": { + "train": { + "mean_centered_corr": -0.019181683659553528, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046132441610097885 + }, + "val": { + "mean_centered_corr": -0.019297907128930092, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06382269412279129 + }, + "test": { + "mean_centered_corr": -0.01942778192460537, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07468391209840775 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 8 + ], + "head_selection_time": "0:00:00.015153" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p001-evaluation/0.toml new file mode 100644 index 000000000..f027505c7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p001-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.95 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..f67da74d3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p001-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23172767733857508, + "mae": 0.12039440870285034, + "r2": 0.6569403409957886, + "score": -0.23172767733857508 + }, + "val": { + "rmse": 0.2350451108570483, + "mae": 0.1278354525566101, + "r2": 0.6309767961502075, + "score": -0.2350451108570483 + }, + "test": { + "rmse": 0.23328519811438406, + "mae": 0.14723573625087738, + "r2": 0.6281104683876038, + "score": -0.23328519811438406 + } + }, + "time": "0:01:31.232910", + "diversity": { + "train": { + "mean_centered_corr": -0.019181683659553528, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046132441610097885 + }, + "val": { + "mean_centered_corr": -0.019297907128930092, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06382269412279129 + }, + "test": { + "mean_centered_corr": -0.01942778192460537, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07468391209840775 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..550a35e9a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,703 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2334628076252813, + "mae": 0.12108320742845535, + "r2": 0.6517835259437561, + "score": -0.2334628076252813 + }, + "val": { + "rmse": 0.23337935525306813, + "mae": 0.12764780223369598, + "r2": 0.6361887454986572, + "score": -0.23337935525306813 + }, + "test": { + "rmse": 0.2363605083000373, + "mae": 0.1529415249824524, + "r2": 0.6182408332824707, + "score": -0.2363605083000373 + } + }, + "time": "0:01:31.232910", + "diversity": { + "train": { + "mean_centered_corr": -0.019181683659553528, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046132441610097885 + }, + "val": { + "mean_centered_corr": -0.019297907128930092, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06382269412279129 + }, + "test": { + "mean_centered_corr": -0.01942778192460537, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07468391209840775 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 8, + 4, + 0, + 12, + 21, + 25, + 1, + 19, + 0, + 4 + ], + "head_selection_time": "0:00:00.159105" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..e90625c54 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2419657538064811, + "mae": 0.12487872689962387, + "r2": 0.6259568929672241, + "score": -0.2419657538064811 + }, + "val": { + "rmse": 0.23774960884328808, + "mae": 0.134882390499115, + "r2": 0.6224357485771179, + "score": -0.23774960884328808 + }, + "test": { + "rmse": 0.23837226942825876, + "mae": 0.1499200165271759, + "r2": 0.6117146015167236, + "score": -0.23837226942825876 + } + }, + "time": "0:01:31.301729", + "diversity": { + "train": { + "mean_centered_corr": -0.01885932870209217, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04668666422367096 + }, + "val": { + "mean_centered_corr": -0.01955920085310936, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06358597427606583 + }, + "test": { + "mean_centered_corr": -0.01645486429333687, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07755117863416672 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015113" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p005-evaluation/0.toml new file mode 100644 index 000000000..da93a9d0c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p005-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.95 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..c44267e3c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p005-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2319513494822566, + "mae": 0.12131161987781525, + "r2": 0.6562777161598206, + "score": -0.2319513494822566 + }, + "val": { + "rmse": 0.23490809371963584, + "mae": 0.12586961686611176, + "r2": 0.6314069032669067, + "score": -0.23490809371963584 + }, + "test": { + "rmse": 0.23175764135614477, + "mae": 0.14321303367614746, + "r2": 0.6329647898674011, + "score": -0.23175764135614477 + } + }, + "time": "0:01:31.301729", + "diversity": { + "train": { + "mean_centered_corr": -0.01885932870209217, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04668666422367096 + }, + "val": { + "mean_centered_corr": -0.01955920085310936, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06358597427606583 + }, + "test": { + "mean_centered_corr": -0.01645486429333687, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07755117863416672 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..7c1e48457 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,701 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23423767040144092, + "mae": 0.12284845113754272, + "r2": 0.6494682431221008, + "score": -0.23423767040144092 + }, + "val": { + "rmse": 0.23309062467822936, + "mae": 0.12788444757461548, + "r2": 0.637088418006897, + "score": -0.23309062467822936 + }, + "test": { + "rmse": 0.2363639835864881, + "mae": 0.15300244092941284, + "r2": 0.6182296276092529, + "score": -0.2363639835864881 + } + }, + "time": "0:01:31.301729", + "diversity": { + "train": { + "mean_centered_corr": -0.01885932870209217, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04668666422367096 + }, + "val": { + "mean_centered_corr": -0.01955920085310936, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06358597427606583 + }, + "test": { + "mean_centered_corr": -0.01645486429333687, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07755117863416672 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 0, + 1, + 19, + 18, + 21, + 25, + 1 + ], + "head_selection_time": "0:00:00.128912" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..fc56d3b75 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23940312390912477, + "mae": 0.12330590933561325, + "r2": 0.6338378190994263, + "score": -0.23940312390912477 + }, + "val": { + "rmse": 0.2385343005293775, + "mae": 0.13736584782600403, + "r2": 0.6199393272399902, + "score": -0.2385343005293775 + }, + "test": { + "rmse": 0.24325157796763505, + "mae": 0.15736712515354156, + "r2": 0.5956560373306274, + "score": -0.24325157796763505 + } + }, + "time": "0:01:31.934812", + "diversity": { + "train": { + "mean_centered_corr": -0.019116338342428207, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047586847096681595 + }, + "val": { + "mean_centered_corr": -0.018940752372145653, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06937787681818008 + }, + "test": { + "mean_centered_corr": -0.01907981187105179, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07969599217176437 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015156" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p01-evaluation/0.toml new file mode 100644 index 000000000..1e0ac3792 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p01-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.95 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..fd189b307 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p01-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23138279613764395, + "mae": 0.12081756442785263, + "r2": 0.6579606533050537, + "score": -0.23138279613764395 + }, + "val": { + "rmse": 0.23465687967262416, + "mae": 0.1256990134716034, + "r2": 0.6321948170661926, + "score": -0.23465687967262416 + }, + "test": { + "rmse": 0.23108455049001858, + "mae": 0.14160698652267456, + "r2": 0.6350936889648438, + "score": -0.23108455049001858 + } + }, + "time": "0:01:31.934812", + "diversity": { + "train": { + "mean_centered_corr": -0.019116338342428207, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047586847096681595 + }, + "val": { + "mean_centered_corr": -0.018940752372145653, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06937787681818008 + }, + "test": { + "mean_centered_corr": -0.01907981187105179, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07969599217176437 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..06c18cf24 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,700 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23336989735039543, + "mae": 0.12107162177562714, + "r2": 0.6520606279373169, + "score": -0.23336989735039543 + }, + "val": { + "rmse": 0.23315395335227065, + "mae": 0.12548291683197021, + "r2": 0.6368911266326904, + "score": -0.23315395335227065 + }, + "test": { + "rmse": 0.2333720363863976, + "mae": 0.14587140083312988, + "r2": 0.6278335452079773, + "score": -0.2333720363863976 + } + }, + "time": "0:01:31.934812", + "diversity": { + "train": { + "mean_centered_corr": -0.019116338342428207, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047586847096681595 + }, + "val": { + "mean_centered_corr": -0.018940752372145653, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06937787681818008 + }, + "test": { + "mean_centered_corr": -0.01907981187105179, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07969599217176437 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 10, + 29, + 19, + 8, + 25, + 21 + ], + "head_selection_time": "0:00:00.115484" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..19c7c3dc8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2238305812388024, + "mae": 0.1228121742606163, + "r2": 0.6799242496490479, + "score": -0.2238305812388024 + }, + "val": { + "rmse": 0.23903748624498683, + "mae": 0.13215485215187073, + "r2": 0.618334174156189, + "score": -0.23903748624498683 + }, + "test": { + "rmse": 0.23879927467488832, + "mae": 0.14591430127620697, + "r2": 0.6103222370147705, + "score": -0.23879927467488832 + } + }, + "time": "0:01:42.466115", + "diversity": { + "train": { + "mean_centered_corr": -0.018108146265149117, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04599704593420029 + }, + "val": { + "mean_centered_corr": -0.017957618460059166, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06543198972940445 + }, + "test": { + "mean_centered_corr": -0.017061514779925346, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09027039259672165 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19 + ], + "head_selection_time": "0:00:00.015148" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p001-evaluation/0.toml new file mode 100644 index 000000000..e3097adcc --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p001-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..2bbcf6ab4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p001-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.21851820244903958, + "mae": 0.11501690745353699, + "r2": 0.6949372887611389, + "score": -0.21851820244903958 + }, + "val": { + "rmse": 0.23496993373476788, + "mae": 0.12609615921974182, + "r2": 0.6312128305435181, + "score": -0.23496993373476788 + }, + "test": { + "rmse": 0.23346087685954336, + "mae": 0.14552420377731323, + "r2": 0.6275501251220703, + "score": -0.23346087685954336 + } + }, + "time": "0:01:42.466115", + "diversity": { + "train": { + "mean_centered_corr": -0.018108146265149117, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04599704593420029 + }, + "val": { + "mean_centered_corr": -0.017957618460059166, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06543198972940445 + }, + "test": { + "mean_centered_corr": -0.017061514779925346, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09027039259672165 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..bb5f598a0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,700 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.22150166122823217, + "mae": 0.11824827641248703, + "r2": 0.6865503191947937, + "score": -0.22150166122823217 + }, + "val": { + "rmse": 0.23308791568735607, + "mae": 0.12669241428375244, + "r2": 0.6370968222618103, + "score": -0.23308791568735607 + }, + "test": { + "rmse": 0.23436618629413886, + "mae": 0.14811882376670837, + "r2": 0.6246559619903564, + "score": -0.23436618629413886 + } + }, + "time": "0:01:42.466115", + "diversity": { + "train": { + "mean_centered_corr": -0.018108146265149117, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04599704593420029 + }, + "val": { + "mean_centered_corr": -0.017957618460059166, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06543198972940445 + }, + "test": { + "mean_centered_corr": -0.017061514779925346, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09027039259672165 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19, + 0, + 22, + 27, + 6, + 21, + 4 + ], + "head_selection_time": "0:00:00.115622" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..562d05f05 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2339843912395287, + "mae": 0.12764866650104523, + "r2": 0.6502258777618408, + "score": -0.2339843912395287 + }, + "val": { + "rmse": 0.23781616129168393, + "mae": 0.13649939000606537, + "r2": 0.6222243309020996, + "score": -0.23781616129168393 + }, + "test": { + "rmse": 0.24546581719417238, + "mae": 0.1635962575674057, + "r2": 0.5882613658905029, + "score": -0.24546581719417238 + } + }, + "time": "0:01:37.486772", + "diversity": { + "train": { + "mean_centered_corr": -0.018235279247164726, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0474671870470047 + }, + "val": { + "mean_centered_corr": -0.01877567730844021, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06657248735427856 + }, + "test": { + "mean_centered_corr": -0.016726242378354073, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08910585194826126 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 29 + ], + "head_selection_time": "0:00:00.015175" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p005-evaluation/0.toml new file mode 100644 index 000000000..77cc09695 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p005-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..b34538bfa --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p005-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22313156758053915, + "mae": 0.11779030412435532, + "r2": 0.6819202899932861, + "score": -0.22313156758053915 + }, + "val": { + "rmse": 0.23492098632735753, + "mae": 0.12972819805145264, + "r2": 0.6313664317131042, + "score": -0.23492098632735753 + }, + "test": { + "rmse": 0.2411043186306478, + "mae": 0.15983712673187256, + "r2": 0.6027631163597107, + "score": -0.2411043186306478 + } + }, + "time": "0:01:37.486772", + "diversity": { + "train": { + "mean_centered_corr": -0.018235279247164726, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0474671870470047 + }, + "val": { + "mean_centered_corr": -0.01877567730844021, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06657248735427856 + }, + "test": { + "mean_centered_corr": -0.016726242378354073, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08910585194826126 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..fa83f78cf --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,704 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22471723040163902, + "mae": 0.11835948377847672, + "r2": 0.6773834228515625, + "score": -0.22471723040163902 + }, + "val": { + "rmse": 0.23341984830196463, + "mae": 0.12756545841693878, + "r2": 0.636062502861023, + "score": -0.23341984830196463 + }, + "test": { + "rmse": 0.24027958886401746, + "mae": 0.15710820257663727, + "r2": 0.6054760813713074, + "score": -0.24027958886401746 + } + }, + "time": "0:01:37.486772", + "diversity": { + "train": { + "mean_centered_corr": -0.018235279247164726, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0474671870470047 + }, + "val": { + "mean_centered_corr": -0.01877567730844021, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06657248735427856 + }, + "test": { + "mean_centered_corr": -0.016726242378354073, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08910585194826126 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 29, + 24, + 4, + 19, + 25, + 7, + 10, + 27, + 22, + 6, + 21 + ], + "head_selection_time": "0:00:00.173692" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..28721b811 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23824613343670828, + "mae": 0.12383664399385452, + "r2": 0.6373684406280518, + "score": -0.23824613343670828 + }, + "val": { + "rmse": 0.23735456374234598, + "mae": 0.13839766383171082, + "r2": 0.6236894130706787, + "score": -0.23735456374234598 + }, + "test": { + "rmse": 0.24576564896936498, + "mae": 0.16598248481750488, + "r2": 0.5872548818588257, + "score": -0.24576564896936498 + } + }, + "time": "0:01:31.618555", + "diversity": { + "train": { + "mean_centered_corr": -0.018080253154039383, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0466311052441597 + }, + "val": { + "mean_centered_corr": -0.019496381282806396, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06081623584032059 + }, + "test": { + "mean_centered_corr": -0.019498975947499275, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08078814297914505 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015235" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p01-evaluation/0.toml new file mode 100644 index 000000000..0746ca938 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p01-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..b68def28e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p01-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23137620304543002, + "mae": 0.12124308198690414, + "r2": 0.65798020362854, + "score": -0.23137620304543002 + }, + "val": { + "rmse": 0.23479386888101392, + "mae": 0.12786690890789032, + "r2": 0.6317652463912964, + "score": -0.23479386888101392 + }, + "test": { + "rmse": 0.23419678588244902, + "mae": 0.1489223688840866, + "r2": 0.6251983642578125, + "score": -0.23419678588244902 + } + }, + "time": "0:01:31.618555", + "diversity": { + "train": { + "mean_centered_corr": -0.018080253154039383, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0466311052441597 + }, + "val": { + "mean_centered_corr": -0.019496381282806396, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06081623584032059 + }, + "test": { + "mean_centered_corr": -0.019498975947499275, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08078814297914505 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..d21e1ac4f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,699 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2340414296102543, + "mae": 0.11944642663002014, + "r2": 0.6500552892684937, + "score": -0.2340414296102543 + }, + "val": { + "rmse": 0.23287923675610403, + "mae": 0.12622950971126556, + "r2": 0.6377463340759277, + "score": -0.23287923675610403 + }, + "test": { + "rmse": 0.23928534680182742, + "mae": 0.15746626257896423, + "r2": 0.6087342500686646, + "score": -0.23928534680182742 + } + }, + "time": "0:01:31.618555", + "diversity": { + "train": { + "mean_centered_corr": -0.018080253154039383, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0466311052441597 + }, + "val": { + "mean_centered_corr": -0.019496381282806396, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06081623584032059 + }, + "test": { + "mean_centered_corr": -0.019498975947499275, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08078814297914505 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 21, + 1, + 20, + 25, + 10 + ], + "head_selection_time": "0:00:00.101709" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..383df988d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2426666319444478, + "mae": 0.12620143592357635, + "r2": 0.6237868070602417, + "score": -0.2426666319444478 + }, + "val": { + "rmse": 0.2389538449467578, + "mae": 0.1335434764623642, + "r2": 0.6186012029647827, + "score": -0.2389538449467578 + }, + "test": { + "rmse": 0.24008690564440552, + "mae": 0.15347233414649963, + "r2": 0.606108546257019, + "score": -0.24008690564440552 + } + }, + "time": "0:01:32.555735", + "diversity": { + "train": { + "mean_centered_corr": -0.019036727026104927, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04819515720009804 + }, + "val": { + "mean_centered_corr": -0.019247591495513916, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0649481937289238 + }, + "test": { + "mean_centered_corr": -0.0215106550604105, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08662677556276321 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015189" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p001-evaluation/0.toml new file mode 100644 index 000000000..636067167 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p001-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..df6f95a5c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p001-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2317936605649109, + "mae": 0.12053651362657547, + "r2": 0.6567448973655701, + "score": -0.2317936605649109 + }, + "val": { + "rmse": 0.234848704054223, + "mae": 0.12717075645923615, + "r2": 0.6315932273864746, + "score": -0.234848704054223 + }, + "test": { + "rmse": 0.23316213384908455, + "mae": 0.14641331136226654, + "r2": 0.6285027265548706, + "score": -0.23316213384908455 + } + }, + "time": "0:01:32.555735", + "diversity": { + "train": { + "mean_centered_corr": -0.019036727026104927, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04819515720009804 + }, + "val": { + "mean_centered_corr": -0.019247591495513916, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0649481937289238 + }, + "test": { + "mean_centered_corr": -0.0215106550604105, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08662677556276321 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..1475d6680 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,701 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2328109289922769, + "mae": 0.12201089411973953, + "r2": 0.6537253856658936, + "score": -0.2328109289922769 + }, + "val": { + "rmse": 0.2336424824585913, + "mae": 0.1279032826423645, + "r2": 0.6353679299354553, + "score": -0.2336424824585913 + }, + "test": { + "rmse": 0.23903173547895684, + "mae": 0.15682479739189148, + "r2": 0.6095632314682007, + "score": -0.23903173547895684 + } + }, + "time": "0:01:32.555735", + "diversity": { + "train": { + "mean_centered_corr": -0.019036727026104927, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04819515720009804 + }, + "val": { + "mean_centered_corr": -0.019247591495513916, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0649481937289238 + }, + "test": { + "mean_centered_corr": -0.0215106550604105, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08662677556276321 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 21, + 19, + 25, + 12, + 25, + 3, + 17 + ], + "head_selection_time": "0:00:00.130557" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..21317ca6a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2394503772198336, + "mae": 0.12332300841808319, + "r2": 0.6336932182312012, + "score": -0.2394503772198336 + }, + "val": { + "rmse": 0.23936406325960766, + "mae": 0.1317620724439621, + "r2": 0.6172905564308167, + "score": -0.23936406325960766 + }, + "test": { + "rmse": 0.23718997553210766, + "mae": 0.1439189463853836, + "r2": 0.6155567169189453, + "score": -0.23718997553210766 + } + }, + "time": "0:01:31.646908", + "diversity": { + "train": { + "mean_centered_corr": -0.01940109021961689, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047315195202827454 + }, + "val": { + "mean_centered_corr": -0.018565993756055832, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06468198448419571 + }, + "test": { + "mean_centered_corr": -0.02023249678313732, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08129923790693283 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 28 + ], + "head_selection_time": "0:00:00.018530" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p005-evaluation/0.toml new file mode 100644 index 000000000..e6f43bff5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p005-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..f5307f814 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p005-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23281691342209276, + "mae": 0.12255796045064926, + "r2": 0.6537076234817505, + "score": -0.23281691342209276 + }, + "val": { + "rmse": 0.23504563388193753, + "mae": 0.12878526747226715, + "r2": 0.6309751272201538, + "score": -0.23504563388193753 + }, + "test": { + "rmse": 0.2337165723322532, + "mae": 0.14694184064865112, + "r2": 0.6267338395118713, + "score": -0.2337165723322532 + } + }, + "time": "0:01:31.646908", + "diversity": { + "train": { + "mean_centered_corr": -0.01940109021961689, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047315195202827454 + }, + "val": { + "mean_centered_corr": -0.018565993756055832, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06468198448419571 + }, + "test": { + "mean_centered_corr": -0.02023249678313732, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08129923790693283 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..eed37620b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,704 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2342020668363707, + "mae": 0.12203548103570938, + "r2": 0.6495747566223145, + "score": -0.2342020668363707 + }, + "val": { + "rmse": 0.23354782526985796, + "mae": 0.12771733105182648, + "r2": 0.6356633305549622, + "score": -0.23354782526985796 + }, + "test": { + "rmse": 0.23343612653897647, + "mae": 0.1471332311630249, + "r2": 0.6276291012763977, + "score": -0.23343612653897647 + } + }, + "time": "0:01:31.646908", + "diversity": { + "train": { + "mean_centered_corr": -0.01940109021961689, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047315195202827454 + }, + "val": { + "mean_centered_corr": -0.018565993756055832, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06468198448419571 + }, + "test": { + "mean_centered_corr": -0.02023249678313732, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08129923790693283 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 28, + 25, + 29, + 19, + 3, + 25, + 4, + 12, + 25, + 27, + 1 + ], + "head_selection_time": "0:00:00.203719" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..7e81581cd --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24306374897467978, + "mae": 0.1263284981250763, + "r2": 0.6225544810295105, + "score": -0.24306374897467978 + }, + "val": { + "rmse": 0.238226555972743, + "mae": 0.13456811010837555, + "r2": 0.6209193468093872, + "score": -0.238226555972743 + }, + "test": { + "rmse": 0.2428780524434567, + "mae": 0.16094788908958435, + "r2": 0.5968968868255615, + "score": -0.2428780524434567 + } + }, + "time": "0:01:29.515084", + "diversity": { + "train": { + "mean_centered_corr": -0.019190935418009758, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04763631150126457 + }, + "val": { + "mean_centered_corr": -0.018849357962608337, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06253517419099808 + }, + "test": { + "mean_centered_corr": -0.020511005073785782, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08172544091939926 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015435" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p01-evaluation/0.toml new file mode 100644 index 000000000..e60ef2e8b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p01-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..13996eb16 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p01-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23280478439090932, + "mae": 0.12093566358089447, + "r2": 0.6537436842918396, + "score": -0.23280478439090932 + }, + "val": { + "rmse": 0.2351138707244049, + "mae": 0.12775011360645294, + "r2": 0.6307608485221863, + "score": -0.2351138707244049 + }, + "test": { + "rmse": 0.23346224116260836, + "mae": 0.14639027416706085, + "r2": 0.6275457739830017, + "score": -0.23346224116260836 + } + }, + "time": "0:01:29.515084", + "diversity": { + "train": { + "mean_centered_corr": -0.019190935418009758, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04763631150126457 + }, + "val": { + "mean_centered_corr": -0.018849357962608337, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06253517419099808 + }, + "test": { + "mean_centered_corr": -0.020511005073785782, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08172544091939926 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..009a3c336 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,702 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23307017461488594, + "mae": 0.12112323194742203, + "r2": 0.6529538035392761, + "score": -0.23307017461488594 + }, + "val": { + "rmse": 0.23345663231013183, + "mae": 0.1278393566608429, + "r2": 0.6359478235244751, + "score": -0.23345663231013183 + }, + "test": { + "rmse": 0.23687685165587416, + "mae": 0.1535518318414688, + "r2": 0.6165710687637329, + "score": -0.23687685165587416 + } + }, + "time": "0:01:29.515084", + "diversity": { + "train": { + "mean_centered_corr": -0.019190935418009758, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04763631150126457 + }, + "val": { + "mean_centered_corr": -0.018849357962608337, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06253517419099808 + }, + "test": { + "mean_centered_corr": -0.020511005073785782, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08172544091939926 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 21, + 25, + 28, + 19, + 3, + 25, + 12, + 25 + ], + "head_selection_time": "0:00:00.145632" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..bbe8c1af4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2285154456765539, + "mae": 0.11967622488737106, + "r2": 0.6663854122161865, + "score": -0.2285154456765539 + }, + "val": { + "rmse": 0.23774841016444426, + "mae": 0.13070452213287354, + "r2": 0.6224395036697388, + "score": -0.23774841016444426 + }, + "test": { + "rmse": 0.24035398870465244, + "mae": 0.15418551862239838, + "r2": 0.605231761932373, + "score": -0.24035398870465244 + } + }, + "time": "0:01:36.865989", + "diversity": { + "train": { + "mean_centered_corr": -0.018838467076420784, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04630524292588234 + }, + "val": { + "mean_centered_corr": -0.01868019439280033, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0651327446103096 + }, + "test": { + "mean_centered_corr": -0.01751791313290596, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08838899433612823 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015153" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p001-evaluation/0.toml new file mode 100644 index 000000000..3499a8693 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p001-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.95 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..215a7ee15 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p001-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22344105424254884, + "mae": 0.11854980140924454, + "r2": 0.6810373663902283, + "score": -0.22344105424254884 + }, + "val": { + "rmse": 0.23512809084220013, + "mae": 0.12858927249908447, + "r2": 0.6307162046432495, + "score": -0.23512809084220013 + }, + "test": { + "rmse": 0.2355998696072441, + "mae": 0.15166118741035461, + "r2": 0.6206940412521362, + "score": -0.2355998696072441 + } + }, + "time": "0:01:36.865989", + "diversity": { + "train": { + "mean_centered_corr": -0.018838467076420784, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04630524292588234 + }, + "val": { + "mean_centered_corr": -0.01868019439280033, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0651327446103096 + }, + "test": { + "mean_centered_corr": -0.01751791313290596, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08838899433612823 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..4b814cf15 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,698 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2240971488045897, + "mae": 0.11656086891889572, + "r2": 0.6791614294052124, + "score": -0.2240971488045897 + }, + "val": { + "rmse": 0.23360989779537025, + "mae": 0.12700146436691284, + "r2": 0.6354696154594421, + "score": -0.23360989779537025 + }, + "test": { + "rmse": 0.24433293207334258, + "mae": 0.16488319635391235, + "r2": 0.5920531153678894, + "score": -0.24433293207334258 + } + }, + "time": "0:01:36.865989", + "diversity": { + "train": { + "mean_centered_corr": -0.018838467076420784, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04630524292588234 + }, + "val": { + "mean_centered_corr": -0.01868019439280033, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0651327446103096 + }, + "test": { + "mean_centered_corr": -0.01751791313290596, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08838899433612823 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 18, + 21, + 25, + 8 + ], + "head_selection_time": "0:00:00.086285" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..8c74b4063 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24133726047749576, + "mae": 0.12366868555545807, + "r2": 0.6278974413871765, + "score": -0.24133726047749576 + }, + "val": { + "rmse": 0.23613982935624828, + "mae": 0.1324111670255661, + "r2": 0.6275312900543213, + "score": -0.23613982935624828 + }, + "test": { + "rmse": 0.23869431760070778, + "mae": 0.15286467969417572, + "r2": 0.6106647253036499, + "score": -0.23869431760070778 + } + }, + "time": "0:01:30.265637", + "diversity": { + "train": { + "mean_centered_corr": -0.019571561366319656, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04648323729634285 + }, + "val": { + "mean_centered_corr": -0.01869725063443184, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06371068954467773 + }, + "test": { + "mean_centered_corr": -0.018871210515499115, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07722875475883484 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015136" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p005-evaluation/0.toml new file mode 100644 index 000000000..bf36690ed --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p005-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.95 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..35d492921 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p005-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23293543424527074, + "mae": 0.12151619791984558, + "r2": 0.6533549427986145, + "score": -0.23293543424527074 + }, + "val": { + "rmse": 0.23518098688395506, + "mae": 0.12663617730140686, + "r2": 0.6305500268936157, + "score": -0.23518098688395506 + }, + "test": { + "rmse": 0.23184570253369863, + "mae": 0.14382076263427734, + "r2": 0.6326858401298523, + "score": -0.23184570253369863 + } + }, + "time": "0:01:30.265637", + "diversity": { + "train": { + "mean_centered_corr": -0.019571561366319656, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04648323729634285 + }, + "val": { + "mean_centered_corr": -0.01869725063443184, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06371068954467773 + }, + "test": { + "mean_centered_corr": -0.018871210515499115, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07722875475883484 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..7d7e41dcc --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,698 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2358128076066539, + "mae": 0.1227794662117958, + "r2": 0.6447380781173706, + "score": -0.2358128076066539 + }, + "val": { + "rmse": 0.23307964467970407, + "mae": 0.12707222998142242, + "r2": 0.6371225714683533, + "score": -0.23307964467970407 + }, + "test": { + "rmse": 0.23920139503937368, + "mae": 0.1561342030763626, + "r2": 0.6090087890625, + "score": -0.23920139503937368 + } + }, + "time": "0:01:30.265637", + "diversity": { + "train": { + "mean_centered_corr": -0.019571561366319656, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04648323729634285 + }, + "val": { + "mean_centered_corr": -0.01869725063443184, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06371068954467773 + }, + "test": { + "mean_centered_corr": -0.018871210515499115, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07722875475883484 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 0, + 8, + 25, + 21 + ], + "head_selection_time": "0:00:00.087431" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..ea48767f9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2313955309804549, + "mae": 0.12607093155384064, + "r2": 0.6579230427742004, + "score": -0.2313955309804549 + }, + "val": { + "rmse": 0.23924971586268343, + "mae": 0.13559652864933014, + "r2": 0.6176561117172241, + "score": -0.23924971586268343 + }, + "test": { + "rmse": 0.23943478014622727, + "mae": 0.1478048413991928, + "r2": 0.6082454323768616, + "score": -0.23943478014622727 + } + }, + "time": "0:01:40.717588", + "diversity": { + "train": { + "mean_centered_corr": -0.018105898052453995, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04715852439403534 + }, + "val": { + "mean_centered_corr": -0.019619733095169067, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06463336944580078 + }, + "test": { + "mean_centered_corr": -0.017858682200312614, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08570578694343567 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 8 + ], + "head_selection_time": "0:00:00.015407" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p01-evaluation/0.toml new file mode 100644 index 000000000..0f3309d70 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p01-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.95 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..62dc1ebc2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p01-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.21986647495899417, + "mae": 0.11594657599925995, + "r2": 0.6911611557006836, + "score": -0.21986647495899417 + }, + "val": { + "rmse": 0.23473991754724038, + "mae": 0.12713272869586945, + "r2": 0.6319344639778137, + "score": -0.23473991754724038 + }, + "test": { + "rmse": 0.23324559206850415, + "mae": 0.14693446457386017, + "r2": 0.628236711025238, + "score": -0.23324559206850415 + } + }, + "time": "0:01:40.717588", + "diversity": { + "train": { + "mean_centered_corr": -0.018105898052453995, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04715852439403534 + }, + "val": { + "mean_centered_corr": -0.019619733095169067, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06463336944580078 + }, + "test": { + "mean_centered_corr": -0.017858682200312614, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08570578694343567 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..d633c837c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,707 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.21944796383995846, + "mae": 0.11816469579935074, + "r2": 0.6923357844352722, + "score": -0.21944796383995846 + }, + "val": { + "rmse": 0.233123281922542, + "mae": 0.127369225025177, + "r2": 0.6369866728782654, + "score": -0.233123281922542 + }, + "test": { + "rmse": 0.2324621021496061, + "mae": 0.14417536556720734, + "r2": 0.6307300925254822, + "score": -0.2324621021496061 + } + }, + "time": "0:01:40.717588", + "diversity": { + "train": { + "mean_centered_corr": -0.018105898052453995, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04715852439403534 + }, + "val": { + "mean_centered_corr": -0.019619733095169067, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06463336944580078 + }, + "test": { + "mean_centered_corr": -0.017858682200312614, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08570578694343567 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 8, + 21, + 31, + 0, + 22, + 14, + 10, + 21, + 31, + 19, + 20, + 4, + 21, + 0 + ], + "head_selection_time": "0:00:00.225657" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..5465e4d4c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2393712066992257, + "mae": 0.12284750491380692, + "r2": 0.6339354515075684, + "score": -0.2393712066992257 + }, + "val": { + "rmse": 0.23604388522704514, + "mae": 0.1302504986524582, + "r2": 0.6278339624404907, + "score": -0.23604388522704514 + }, + "test": { + "rmse": 0.2413116661749965, + "mae": 0.15745559334754944, + "r2": 0.6020795702934265, + "score": -0.2413116661749965 + } + }, + "time": "0:01:31.102179", + "diversity": { + "train": { + "mean_centered_corr": -0.019335079938173294, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045941311866045 + }, + "val": { + "mean_centered_corr": -0.019422883167862892, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05757637694478035 + }, + "test": { + "mean_centered_corr": -0.018836498260498047, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07515785843133926 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015295" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p001-evaluation/0.toml new file mode 100644 index 000000000..49391d2ba --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p001-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..a1d1c3a33 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p001-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23172434955038107, + "mae": 0.11937171965837479, + "r2": 0.6569501161575317, + "score": -0.23172434955038107 + }, + "val": { + "rmse": 0.23468678716863678, + "mae": 0.12593677639961243, + "r2": 0.6321010589599609, + "score": -0.23468678716863678 + }, + "test": { + "rmse": 0.23217868905016578, + "mae": 0.14450614154338837, + "r2": 0.6316299438476562, + "score": -0.23217868905016578 + } + }, + "time": "0:01:31.102179", + "diversity": { + "train": { + "mean_centered_corr": -0.019335079938173294, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045941311866045 + }, + "val": { + "mean_centered_corr": -0.019422883167862892, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05757637694478035 + }, + "test": { + "mean_centered_corr": -0.018836498260498047, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07515785843133926 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..ede0cf086 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,701 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2339039041660179, + "mae": 0.12130481749773026, + "r2": 0.6504664421081543, + "score": -0.2339039041660179 + }, + "val": { + "rmse": 0.23298097729336806, + "mae": 0.12618261575698853, + "r2": 0.6374297142028809, + "score": -0.23298097729336806 + }, + "test": { + "rmse": 0.23673558379896892, + "mae": 0.15379810333251953, + "r2": 0.6170282959938049, + "score": -0.23673558379896892 + } + }, + "time": "0:01:31.102179", + "diversity": { + "train": { + "mean_centered_corr": -0.019335079938173294, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045941311866045 + }, + "val": { + "mean_centered_corr": -0.019422883167862892, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05757637694478035 + }, + "test": { + "mean_centered_corr": -0.018836498260498047, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07515785843133926 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 1, + 16, + 0, + 21, + 25, + 20, + 1 + ], + "head_selection_time": "0:00:00.129256" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..ee07af13d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2402521219909073, + "mae": 0.12393063306808472, + "r2": 0.6312361359596252, + "score": -0.2402521219909073 + }, + "val": { + "rmse": 0.2369960693297996, + "mae": 0.1321815848350525, + "r2": 0.6248252987861633, + "score": -0.2369960693297996 + }, + "test": { + "rmse": 0.2449247661219165, + "mae": 0.16224433481693268, + "r2": 0.5900744199752808, + "score": -0.2449247661219165 + } + }, + "time": "0:01:31.861899", + "diversity": { + "train": { + "mean_centered_corr": -0.01910976506769657, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04582935944199562 + }, + "val": { + "mean_centered_corr": -0.01908961497247219, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06326822191476822 + }, + "test": { + "mean_centered_corr": -0.02014167793095112, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08288853615522385 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015320" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p005-evaluation/0.toml new file mode 100644 index 000000000..968761935 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p005-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..fe64ac728 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p005-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23193927958696722, + "mae": 0.12050655484199524, + "r2": 0.6563134789466858, + "score": -0.23193927958696722 + }, + "val": { + "rmse": 0.23531282668072034, + "mae": 0.1256631463766098, + "r2": 0.6301356554031372, + "score": -0.23531282668072034 + }, + "test": { + "rmse": 0.2329136030473919, + "mae": 0.1449238359928131, + "r2": 0.6292942762374878, + "score": -0.2329136030473919 + } + }, + "time": "0:01:31.861899", + "diversity": { + "train": { + "mean_centered_corr": -0.01910976506769657, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04582935944199562 + }, + "val": { + "mean_centered_corr": -0.01908961497247219, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06326822191476822 + }, + "test": { + "mean_centered_corr": -0.02014167793095112, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08288853615522385 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..13bb585de --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,699 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23367056683958856, + "mae": 0.12022486329078674, + "r2": 0.6511635184288025, + "score": -0.23367056683958856 + }, + "val": { + "rmse": 0.23320698567170814, + "mae": 0.12736979126930237, + "r2": 0.6367259621620178, + "score": -0.23320698567170814 + }, + "test": { + "rmse": 0.24367480279619608, + "mae": 0.16372117400169373, + "r2": 0.5942478179931641, + "score": -0.24367480279619608 + } + }, + "time": "0:01:31.861899", + "diversity": { + "train": { + "mean_centered_corr": -0.01910976506769657, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04582935944199562 + }, + "val": { + "mean_centered_corr": -0.01908961497247219, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06326822191476822 + }, + "test": { + "mean_centered_corr": -0.02014167793095112, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08288853615522385 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 0, + 21, + 18, + 25, + 10 + ], + "head_selection_time": "0:00:00.101265" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..e5e21eca5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23893874555528044, + "mae": 0.13176758587360382, + "r2": 0.6352569460868835, + "score": -0.23893874555528044 + }, + "val": { + "rmse": 0.23723977382385666, + "mae": 0.13325777649879456, + "r2": 0.6240533590316772, + "score": -0.23723977382385666 + }, + "test": { + "rmse": 0.24324646285162105, + "mae": 0.15948642790317535, + "r2": 0.5956730842590332, + "score": -0.24324646285162105 + } + }, + "time": "0:01:30.961009", + "diversity": { + "train": { + "mean_centered_corr": -0.0197697002440691, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046830255538225174 + }, + "val": { + "mean_centered_corr": -0.01885363645851612, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06196551397442818 + }, + "test": { + "mean_centered_corr": -0.01950950175523758, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07843607664108276 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0 + ], + "head_selection_time": "0:00:00.015095" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p01-evaluation/0.toml new file mode 100644 index 000000000..c819006f0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p01-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..4f6082808 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p01-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23177381131405664, + "mae": 0.11973857134580612, + "r2": 0.6568037271499634, + "score": -0.23177381131405664 + }, + "val": { + "rmse": 0.23489533520890316, + "mae": 0.12537376582622528, + "r2": 0.6314469575881958, + "score": -0.23489533520890316 + }, + "test": { + "rmse": 0.23252367146541786, + "mae": 0.14447112381458282, + "r2": 0.6305344700813293, + "score": -0.23252367146541786 + } + }, + "time": "0:01:30.961009", + "diversity": { + "train": { + "mean_centered_corr": -0.0197697002440691, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046830255538225174 + }, + "val": { + "mean_centered_corr": -0.01885363645851612, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06196551397442818 + }, + "test": { + "mean_centered_corr": -0.01950950175523758, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07843607664108276 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..c181f9775 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,698 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23515993444908453, + "mae": 0.12149348109960556, + "r2": 0.6467025279998779, + "score": -0.23515993444908453 + }, + "val": { + "rmse": 0.23277517922062366, + "mae": 0.1261647343635559, + "r2": 0.6380699872970581, + "score": -0.23277517922062366 + }, + "test": { + "rmse": 0.239397685367944, + "mae": 0.1571250855922699, + "r2": 0.608366847038269, + "score": -0.239397685367944 + } + }, + "time": "0:01:30.961009", + "diversity": { + "train": { + "mean_centered_corr": -0.0197697002440691, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046830255538225174 + }, + "val": { + "mean_centered_corr": -0.01885363645851612, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06196551397442818 + }, + "test": { + "mean_centered_corr": -0.01950950175523758, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07843607664108276 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0, + 4, + 21, + 25, + 1 + ], + "head_selection_time": "0:00:00.086399" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..6322e1094 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.23146582564861154, + "mae": 0.12501563131809235, + "r2": 0.6577152013778687, + "score": -0.23146582564861154 + }, + "val": { + "rmse": 0.23843676541966466, + "mae": 0.14031149446964264, + "r2": 0.6202501058578491, + "score": -0.23843676541966466 + }, + "test": { + "rmse": 0.2470445431022663, + "mae": 0.16841350495815277, + "r2": 0.5829480886459351, + "score": -0.2470445431022663 + } + }, + "time": "0:01:44.046551", + "diversity": { + "train": { + "mean_centered_corr": -0.01830325461924076, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04772296920418739 + }, + "val": { + "mean_centered_corr": -0.019104016944766045, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06741341203451157 + }, + "test": { + "mean_centered_corr": -0.017741652205586433, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09074985980987549 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25 + ], + "head_selection_time": "0:00:00.015187" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p001-evaluation/0.toml new file mode 100644 index 000000000..86978b7b8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p001-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..511d952ad --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p001-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2188970803621353, + "mae": 0.11487266421318054, + "r2": 0.6938785314559937, + "score": -0.2188970803621353 + }, + "val": { + "rmse": 0.23493513883666117, + "mae": 0.12670956552028656, + "r2": 0.6313220262527466, + "score": -0.23493513883666117 + }, + "test": { + "rmse": 0.2324465730511325, + "mae": 0.14363469183444977, + "r2": 0.6307794451713562, + "score": -0.2324465730511325 + } + }, + "time": "0:01:44.046551", + "diversity": { + "train": { + "mean_centered_corr": -0.01830325461924076, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04772296920418739 + }, + "val": { + "mean_centered_corr": -0.019104016944766045, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06741341203451157 + }, + "test": { + "mean_centered_corr": -0.017741652205586433, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09074985980987549 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..2b2b0f319 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,700 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2202455368136436, + "mae": 0.11561281979084015, + "r2": 0.6900953054428101, + "score": -0.2202455368136436 + }, + "val": { + "rmse": 0.23360278549070665, + "mae": 0.12652066349983215, + "r2": 0.6354918479919434, + "score": -0.23360278549070665 + }, + "test": { + "rmse": 0.23177048418825977, + "mae": 0.1428041160106659, + "r2": 0.6329240798950195, + "score": -0.23177048418825977 + } + }, + "time": "0:01:44.046551", + "diversity": { + "train": { + "mean_centered_corr": -0.01830325461924076, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04772296920418739 + }, + "val": { + "mean_centered_corr": -0.019104016944766045, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06741341203451157 + }, + "test": { + "mean_centered_corr": -0.017741652205586433, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09074985980987549 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25, + 19, + 0, + 3, + 21, + 1, + 20 + ], + "head_selection_time": "0:00:00.116231" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..7777a7a3e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24224300671355675, + "mae": 0.12633994221687317, + "r2": 0.6250991821289062, + "score": -0.24224300671355675 + }, + "val": { + "rmse": 0.23790953518728727, + "mae": 0.1355382353067398, + "r2": 0.6219276189804077, + "score": -0.23790953518728727 + }, + "test": { + "rmse": 0.23821708722105353, + "mae": 0.15011507272720337, + "r2": 0.6122199892997742, + "score": -0.23821708722105353 + } + }, + "time": "0:01:29.035550", + "diversity": { + "train": { + "mean_centered_corr": -0.0192310381680727, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04764547199010849 + }, + "val": { + "mean_centered_corr": -0.018086779862642288, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0648212805390358 + }, + "test": { + "mean_centered_corr": -0.01891481876373291, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08217111229896545 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015344" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p005-evaluation/0.toml new file mode 100644 index 000000000..ddec6331a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p005-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..f9245b832 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p005-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23216875702743484, + "mae": 0.12165454775094986, + "r2": 0.6556330919265747, + "score": -0.23216875702743484 + }, + "val": { + "rmse": 0.2350836134726586, + "mae": 0.128606379032135, + "r2": 0.6308558583259583, + "score": -0.2350836134726586 + }, + "test": { + "rmse": 0.23409832677668932, + "mae": 0.1483079195022583, + "r2": 0.6255134344100952, + "score": -0.23409832677668932 + } + }, + "time": "0:01:29.035550", + "diversity": { + "train": { + "mean_centered_corr": -0.0192310381680727, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04764547199010849 + }, + "val": { + "mean_centered_corr": -0.018086779862642288, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0648212805390358 + }, + "test": { + "mean_centered_corr": -0.01891481876373291, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08217111229896545 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..1c6bc379c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,703 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23386279403869487, + "mae": 0.1229424923658371, + "r2": 0.6505893468856812, + "score": -0.23386279403869487 + }, + "val": { + "rmse": 0.23395800051802765, + "mae": 0.1274479776620865, + "r2": 0.634382426738739, + "score": -0.23395800051802765 + }, + "test": { + "rmse": 0.23212124110153423, + "mae": 0.1446542590856552, + "r2": 0.6318122148513794, + "score": -0.23212124110153423 + } + }, + "time": "0:01:29.035550", + "diversity": { + "train": { + "mean_centered_corr": -0.0192310381680727, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04764547199010849 + }, + "val": { + "mean_centered_corr": -0.018086779862642288, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0648212805390358 + }, + "test": { + "mean_centered_corr": -0.01891481876373291, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08217111229896545 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 28, + 19, + 21, + 0, + 24, + 29, + 10, + 16, + 3 + ], + "head_selection_time": "0:00:00.157884" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..db37da897 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23859920550431532, + "mae": 0.12155521661043167, + "r2": 0.6362928152084351, + "score": -0.23859920550431532 + }, + "val": { + "rmse": 0.23752709943108333, + "mae": 0.1321943998336792, + "r2": 0.6231421232223511, + "score": -0.23752709943108333 + }, + "test": { + "rmse": 0.23270363118336854, + "mae": 0.14155620336532593, + "r2": 0.6299623250961304, + "score": -0.23270363118336854 + } + }, + "time": "0:01:30.732738", + "diversity": { + "train": { + "mean_centered_corr": -0.018462583422660828, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047156888991594315 + }, + "val": { + "mean_centered_corr": -0.017515555024147034, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06680545955896378 + }, + "test": { + "mean_centered_corr": -0.017912454903125763, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08376385271549225 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 28 + ], + "head_selection_time": "0:00:00.015330" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p01-evaluation/0.toml new file mode 100644 index 000000000..0dfe890fe --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p01-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..5e260f037 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p01-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23201801570565175, + "mae": 0.1213398203253746, + "r2": 0.6560801267623901, + "score": -0.23201801570565175 + }, + "val": { + "rmse": 0.23493006464565203, + "mae": 0.1288696676492691, + "r2": 0.6313378810882568, + "score": -0.23493006464565203 + }, + "test": { + "rmse": 0.23367776476223437, + "mae": 0.14683480560779572, + "r2": 0.6268577575683594, + "score": -0.23367776476223437 + } + }, + "time": "0:01:30.732738", + "diversity": { + "train": { + "mean_centered_corr": -0.018462583422660828, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047156888991594315 + }, + "val": { + "mean_centered_corr": -0.017515555024147034, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06680545955896378 + }, + "test": { + "mean_centered_corr": -0.017912454903125763, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08376385271549225 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..4e3ae064e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,705 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.232872877996383, + "mae": 0.12275310605764389, + "r2": 0.653541088104248, + "score": -0.232872877996383 + }, + "val": { + "rmse": 0.23302229095267976, + "mae": 0.12814900279045105, + "r2": 0.6373011469841003, + "score": -0.23302229095267976 + }, + "test": { + "rmse": 0.2330139696533979, + "mae": 0.146689772605896, + "r2": 0.6289746761322021, + "score": -0.2330139696533979 + } + }, + "time": "0:01:30.732738", + "diversity": { + "train": { + "mean_centered_corr": -0.018462583422660828, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047156888991594315 + }, + "val": { + "mean_centered_corr": -0.017515555024147034, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06680545955896378 + }, + "test": { + "mean_centered_corr": -0.017912454903125763, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08376385271549225 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 28, + 19, + 25, + 4, + 12, + 24, + 0, + 21, + 25, + 29, + 19, + 4 + ], + "head_selection_time": "0:00:00.191270" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..f019e5b6f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2252595034159999, + "mae": 0.12715505063533783, + "r2": 0.6758245229721069, + "score": -0.2252595034159999 + }, + "val": { + "rmse": 0.23834036265829958, + "mae": 0.133585587143898, + "r2": 0.6205570697784424, + "score": -0.23834036265829958 + }, + "test": { + "rmse": 0.2355387485424496, + "mae": 0.14495041966438293, + "r2": 0.6208908557891846, + "score": -0.2355387485424496 + } + }, + "time": "0:02:14.105851", + "diversity": { + "train": { + "mean_centered_corr": -0.018248364329338074, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045475468039512634 + }, + "val": { + "mean_centered_corr": -0.01828787662088871, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06622428447008133 + }, + "test": { + "mean_centered_corr": -0.01435924507677555, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09353811293840408 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 19 + ], + "head_selection_time": "0:00:00.015604" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p001-evaluation/0.toml new file mode 100644 index 000000000..b350c1c90 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p001-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.95 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..d66ed52f1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p001-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22275971635451677, + "mae": 0.11979660391807556, + "r2": 0.6829795837402344, + "score": -0.22275971635451677 + }, + "val": { + "rmse": 0.23522175576712184, + "mae": 0.1303437203168869, + "r2": 0.6304218769073486, + "score": -0.23522175576712184 + }, + "test": { + "rmse": 0.2392933877459825, + "mae": 0.1575920134782791, + "r2": 0.6087079644203186, + "score": -0.2392933877459825 + } + }, + "time": "0:02:14.105851", + "diversity": { + "train": { + "mean_centered_corr": -0.018248364329338074, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045475468039512634 + }, + "val": { + "mean_centered_corr": -0.01828787662088871, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06622428447008133 + }, + "test": { + "mean_centered_corr": -0.01435924507677555, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09353811293840408 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..7bbd01f02 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,701 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22522487065919897, + "mae": 0.1186138316988945, + "r2": 0.6759241819381714, + "score": -0.22522487065919897 + }, + "val": { + "rmse": 0.2328865391175455, + "mae": 0.12701237201690674, + "r2": 0.6377235651016235, + "score": -0.2328865391175455 + }, + "test": { + "rmse": 0.23281656140106766, + "mae": 0.14357610046863556, + "r2": 0.6296030879020691, + "score": -0.23281656140106766 + } + }, + "time": "0:02:14.105851", + "diversity": { + "train": { + "mean_centered_corr": -0.018248364329338074, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045475468039512634 + }, + "val": { + "mean_centered_corr": -0.01828787662088871, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06622428447008133 + }, + "test": { + "mean_centered_corr": -0.01435924507677555, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09353811293840408 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 19, + 8, + 4, + 29, + 7, + 8, + 1, + 21 + ], + "head_selection_time": "0:00:00.131174" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..4d9024b29 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.229118968956047, + "mae": 0.12437356263399124, + "r2": 0.6646209359169006, + "score": -0.229118968956047 + }, + "val": { + "rmse": 0.23782636654662812, + "mae": 0.12977644801139832, + "r2": 0.6221919059753418, + "score": -0.23782636654662812 + }, + "test": { + "rmse": 0.23905601555002126, + "mae": 0.14474956691265106, + "r2": 0.6094838976860046, + "score": -0.23905601555002126 + } + }, + "time": "0:02:16.774478", + "diversity": { + "train": { + "mean_centered_corr": -0.018586045131087303, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045367300510406494 + }, + "val": { + "mean_centered_corr": -0.019293533638119698, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07069077342748642 + }, + "test": { + "mean_centered_corr": -0.01720191352069378, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09131180495023727 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 8 + ], + "head_selection_time": "0:00:00.015326" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p005-evaluation/0.toml new file mode 100644 index 000000000..8fc7a8082 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p005-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.95 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..d5fd99862 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p005-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.21912932715302513, + "mae": 0.11480405181646347, + "r2": 0.6932286024093628, + "score": -0.21912932715302513 + }, + "val": { + "rmse": 0.2351262925795415, + "mae": 0.12626133859157562, + "r2": 0.6307218074798584, + "score": -0.2351262925795415 + }, + "test": { + "rmse": 0.23336620188006213, + "mae": 0.14624115824699402, + "r2": 0.627852201461792, + "score": -0.23336620188006213 + } + }, + "time": "0:02:16.774478", + "diversity": { + "train": { + "mean_centered_corr": -0.018586045131087303, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045367300510406494 + }, + "val": { + "mean_centered_corr": -0.019293533638119698, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07069077342748642 + }, + "test": { + "mean_centered_corr": -0.01720191352069378, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09131180495023727 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..dff01cfa8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,700 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2210335747342936, + "mae": 0.11760202050209045, + "r2": 0.6878736615180969, + "score": -0.2210335747342936 + }, + "val": { + "rmse": 0.2332281025812143, + "mae": 0.12575659155845642, + "r2": 0.6366601586341858, + "score": -0.2332281025812143 + }, + "test": { + "rmse": 0.23533248824072742, + "mae": 0.14844603836536407, + "r2": 0.6215544939041138, + "score": -0.23533248824072742 + } + }, + "time": "0:02:16.774478", + "diversity": { + "train": { + "mean_centered_corr": -0.018586045131087303, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045367300510406494 + }, + "val": { + "mean_centered_corr": -0.019293533638119698, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07069077342748642 + }, + "test": { + "mean_centered_corr": -0.01720191352069378, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09131180495023727 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 8, + 27, + 19, + 0, + 25, + 21, + 4 + ], + "head_selection_time": "0:00:00.117123" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..50c13353b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2242175374679477, + "mae": 0.12167605012655258, + "r2": 0.6788166761398315, + "score": -0.2242175374679477 + }, + "val": { + "rmse": 0.2383424727161954, + "mae": 0.1329953521490097, + "r2": 0.6205503940582275, + "score": -0.2383424727161954 + }, + "test": { + "rmse": 0.23929541934545473, + "mae": 0.1496122032403946, + "r2": 0.6087013483047485, + "score": -0.23929541934545473 + } + }, + "time": "0:02:17.580078", + "diversity": { + "train": { + "mean_centered_corr": -0.018310561776161194, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04597850888967514 + }, + "val": { + "mean_centered_corr": -0.01918131671845913, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06879474967718124 + }, + "test": { + "mean_centered_corr": -0.016168270260095596, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08937344700098038 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015136" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p01-evaluation/0.toml new file mode 100644 index 000000000..6b87f0411 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p01-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.95 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..88bb0714f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p01-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.21875939178741377, + "mae": 0.11528072506189346, + "r2": 0.6942634582519531, + "score": -0.21875939178741377 + }, + "val": { + "rmse": 0.23509613200143026, + "mae": 0.12603282928466797, + "r2": 0.6308165788650513, + "score": -0.23509613200143026 + }, + "test": { + "rmse": 0.23336054282826887, + "mae": 0.14659926295280457, + "r2": 0.6278702020645142, + "score": -0.23336054282826887 + } + }, + "time": "0:02:17.580078", + "diversity": { + "train": { + "mean_centered_corr": -0.018310561776161194, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04597850888967514 + }, + "val": { + "mean_centered_corr": -0.01918131671845913, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06879474967718124 + }, + "test": { + "mean_centered_corr": -0.016168270260095596, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08937344700098038 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..5d88fb9cb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,699 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.21918426561919716, + "mae": 0.11884808540344238, + "r2": 0.6930747032165527, + "score": -0.21918426561919716 + }, + "val": { + "rmse": 0.23335380606300898, + "mae": 0.12806729972362518, + "r2": 0.6362683773040771, + "score": -0.23335380606300898 + }, + "test": { + "rmse": 0.23440792329336166, + "mae": 0.1488981395959854, + "r2": 0.62452232837677, + "score": -0.23440792329336166 + } + }, + "time": "0:02:17.580078", + "diversity": { + "train": { + "mean_centered_corr": -0.018310561776161194, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04597850888967514 + }, + "val": { + "mean_centered_corr": -0.01918131671845913, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06879474967718124 + }, + "test": { + "mean_centered_corr": -0.016168270260095596, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08937344700098038 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 4, + 25, + 19, + 21, + 8, + 31 + ], + "head_selection_time": "0:00:00.101201" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..c5a66d7c9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2362965728252335, + "mae": 0.12153194099664688, + "r2": 0.643278956413269, + "score": -0.2362965728252335 + }, + "val": { + "rmse": 0.2357278561917395, + "mae": 0.1297774612903595, + "r2": 0.628829836845398, + "score": -0.2357278561917395 + }, + "test": { + "rmse": 0.24177902137278479, + "mae": 0.15532447397708893, + "r2": 0.6005367636680603, + "score": -0.24177902137278479 + } + }, + "time": "0:02:03.045766", + "diversity": { + "train": { + "mean_centered_corr": -0.018870852887630463, + "mean_pairwise_disagreement": NaN, + "member_std": 0.044821128249168396 + }, + "val": { + "mean_centered_corr": -0.018474284559488297, + "mean_pairwise_disagreement": NaN, + "member_std": 0.057242587208747864 + }, + "test": { + "mean_centered_corr": -0.01819882169365883, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08083672821521759 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015341" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p001-evaluation/0.toml new file mode 100644 index 000000000..2fdcae837 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p001-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..c1382027a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p001-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.22965888604190204, + "mae": 0.12082202732563019, + "r2": 0.6630383729934692, + "score": -0.22965888604190204 + }, + "val": { + "rmse": 0.2346724530004704, + "mae": 0.12799137830734253, + "r2": 0.6321460008621216, + "score": -0.2346724530004704 + }, + "test": { + "rmse": 0.23472572946107031, + "mae": 0.15012341737747192, + "r2": 0.6235034465789795, + "score": -0.23472572946107031 + } + }, + "time": "0:02:03.045766", + "diversity": { + "train": { + "mean_centered_corr": -0.018870852887630463, + "mean_pairwise_disagreement": NaN, + "member_std": 0.044821128249168396 + }, + "val": { + "mean_centered_corr": -0.018474284559488297, + "mean_pairwise_disagreement": NaN, + "member_std": 0.057242587208747864 + }, + "test": { + "mean_centered_corr": -0.01819882169365883, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08083672821521759 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..ec97c5157 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,699 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23456546673625236, + "mae": 0.12042497098445892, + "r2": 0.6484864950180054, + "score": -0.23456546673625236 + }, + "val": { + "rmse": 0.2330191974784263, + "mae": 0.12631137669086456, + "r2": 0.6373107433319092, + "score": -0.2330191974784263 + }, + "test": { + "rmse": 0.24024111265333, + "mae": 0.15944169461727142, + "r2": 0.6056023836135864, + "score": -0.24024111265333 + } + }, + "time": "0:02:03.045766", + "diversity": { + "train": { + "mean_centered_corr": -0.018870852887630463, + "mean_pairwise_disagreement": NaN, + "member_std": 0.044821128249168396 + }, + "val": { + "mean_centered_corr": -0.018474284559488297, + "mean_pairwise_disagreement": NaN, + "member_std": 0.057242587208747864 + }, + "test": { + "mean_centered_corr": -0.01819882169365883, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08083672821521759 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 4, + 21, + 0, + 10, + 1, + 28 + ], + "head_selection_time": "0:00:00.101866" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..485c6f813 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23715487802399932, + "mae": 0.12212099879980087, + "r2": 0.6406828165054321, + "score": -0.23715487802399932 + }, + "val": { + "rmse": 0.2361938000242818, + "mae": 0.130824014544487, + "r2": 0.6273610591888428, + "score": -0.2361938000242818 + }, + "test": { + "rmse": 0.23968299921179284, + "mae": 0.15190771222114563, + "r2": 0.6074327826499939, + "score": -0.23968299921179284 + } + }, + "time": "0:02:03.488922", + "diversity": { + "train": { + "mean_centered_corr": -0.019381657242774963, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04552893713116646 + }, + "val": { + "mean_centered_corr": -0.01903798058629036, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06320050358772278 + }, + "test": { + "mean_centered_corr": -0.018483227118849754, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08418284356594086 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015490" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p005-evaluation/0.toml new file mode 100644 index 000000000..705351034 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p005-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..a4ae477f4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p005-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23023425311605375, + "mae": 0.11950831860303879, + "r2": 0.6613479256629944, + "score": -0.23023425311605375 + }, + "val": { + "rmse": 0.23514965307924723, + "mae": 0.12576493620872498, + "r2": 0.6306484341621399, + "score": -0.23514965307924723 + }, + "test": { + "rmse": 0.23159657135010447, + "mae": 0.1426416039466858, + "r2": 0.6334748268127441, + "score": -0.23159657135010447 + } + }, + "time": "0:02:03.488922", + "diversity": { + "train": { + "mean_centered_corr": -0.019381657242774963, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04552893713116646 + }, + "val": { + "mean_centered_corr": -0.01903798058629036, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06320050358772278 + }, + "test": { + "mean_centered_corr": -0.018483227118849754, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08418284356594086 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..ccb8e9619 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,697 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23299274539756, + "mae": 0.12110579758882523, + "r2": 0.6531843543052673, + "score": -0.23299274539756 + }, + "val": { + "rmse": 0.23281693742350693, + "mae": 0.1263430267572403, + "r2": 0.6379401087760925, + "score": -0.23281693742350693 + }, + "test": { + "rmse": 0.23764734685399194, + "mae": 0.15499013662338257, + "r2": 0.6140726804733276, + "score": -0.23764734685399194 + } + }, + "time": "0:02:03.488922", + "diversity": { + "train": { + "mean_centered_corr": -0.019381657242774963, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04552893713116646 + }, + "val": { + "mean_centered_corr": -0.01903798058629036, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06320050358772278 + }, + "test": { + "mean_centered_corr": -0.018483227118849754, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08418284356594086 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 4, + 0, + 25, + 21 + ], + "head_selection_time": "0:00:00.074365" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..a5b17ed41 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23937254509775088, + "mae": 0.12237884849309921, + "r2": 0.6339312791824341, + "score": -0.23937254509775088 + }, + "val": { + "rmse": 0.236445130001989, + "mae": 0.13012416660785675, + "r2": 0.6265676021575928, + "score": -0.236445130001989 + }, + "test": { + "rmse": 0.2384320235467145, + "mae": 0.14947566390037537, + "r2": 0.6115199327468872, + "score": -0.2384320235467145 + } + }, + "time": "0:02:02.511800", + "diversity": { + "train": { + "mean_centered_corr": -0.01890525035560131, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0457611083984375 + }, + "val": { + "mean_centered_corr": -0.019421136006712914, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06420256942510605 + }, + "test": { + "mean_centered_corr": -0.01877988502383232, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08486068248748779 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015361" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p01-evaluation/0.toml new file mode 100644 index 000000000..8b4c4cda1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p01-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..a464d6f76 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p01-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23134854060109078, + "mae": 0.11915364116430283, + "r2": 0.6580619812011719, + "score": -0.23134854060109078 + }, + "val": { + "rmse": 0.23526017405140137, + "mae": 0.12479399889707565, + "r2": 0.6303011775016785, + "score": -0.23526017405140137 + }, + "test": { + "rmse": 0.23237540477719482, + "mae": 0.14426252245903015, + "r2": 0.6310054659843445, + "score": -0.23237540477719482 + } + }, + "time": "0:02:02.511800", + "diversity": { + "train": { + "mean_centered_corr": -0.01890525035560131, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0457611083984375 + }, + "val": { + "mean_centered_corr": -0.019421136006712914, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06420256942510605 + }, + "test": { + "mean_centered_corr": -0.01877988502383232, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08486068248748779 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..a01a45660 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,699 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23389260398134046, + "mae": 0.12011703103780746, + "r2": 0.6505002379417419, + "score": -0.23389260398134046 + }, + "val": { + "rmse": 0.23288295595018185, + "mae": 0.1259579211473465, + "r2": 0.6377347707748413, + "score": -0.23288295595018185 + }, + "test": { + "rmse": 0.2378111877355073, + "mae": 0.15456625819206238, + "r2": 0.6135403513908386, + "score": -0.2378111877355073 + } + }, + "time": "0:02:02.511800", + "diversity": { + "train": { + "mean_centered_corr": -0.01890525035560131, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0457611083984375 + }, + "val": { + "mean_centered_corr": -0.019421136006712914, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06420256942510605 + }, + "test": { + "mean_centered_corr": -0.01877988502383232, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08486068248748779 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 4, + 16, + 19, + 10, + 25, + 1 + ], + "head_selection_time": "0:00:00.102182" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..050fbfdcd --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24201867156469675, + "mae": 0.1283414661884308, + "r2": 0.6257932186126709, + "score": -0.24201867156469675 + }, + "val": { + "rmse": 0.23792789399055542, + "mae": 0.13413427770137787, + "r2": 0.6218692660331726, + "score": -0.23792789399055542 + }, + "test": { + "rmse": 0.23833634568163614, + "mae": 0.15245570242404938, + "r2": 0.6118316650390625, + "score": -0.23833634568163614 + } + }, + "time": "0:02:02.915603", + "diversity": { + "train": { + "mean_centered_corr": -0.018602978438138962, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04584116488695145 + }, + "val": { + "mean_centered_corr": -0.01837269775569439, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06368750333786011 + }, + "test": { + "mean_centered_corr": -0.019264845177531242, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08078610152006149 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015414" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p001-evaluation/0.toml new file mode 100644 index 000000000..a3f9ae4bb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p001-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..8ce5fa5b2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p001-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23062440497365552, + "mae": 0.12134777754545212, + "r2": 0.6601991653442383, + "score": -0.23062440497365552 + }, + "val": { + "rmse": 0.23522375918391628, + "mae": 0.1281193345785141, + "r2": 0.6304156184196472, + "score": -0.23522375918391628 + }, + "test": { + "rmse": 0.23361716934488255, + "mae": 0.14736692607402802, + "r2": 0.6270512938499451, + "score": -0.23361716934488255 + } + }, + "time": "0:02:02.915603", + "diversity": { + "train": { + "mean_centered_corr": -0.018602978438138962, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04584116488695145 + }, + "val": { + "mean_centered_corr": -0.01837269775569439, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06368750333786011 + }, + "test": { + "mean_centered_corr": -0.019264845177531242, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08078610152006149 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..b18c6b5b4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,699 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23198909697727824, + "mae": 0.12181763350963593, + "r2": 0.6561658382415771, + "score": -0.23198909697727824 + }, + "val": { + "rmse": 0.23351789154964117, + "mae": 0.12995123863220215, + "r2": 0.6357567310333252, + "score": -0.23351789154964117 + }, + "test": { + "rmse": 0.2420072500053204, + "mae": 0.1604735553264618, + "r2": 0.5997822284698486, + "score": -0.2420072500053204 + } + }, + "time": "0:02:02.915603", + "diversity": { + "train": { + "mean_centered_corr": -0.018602978438138962, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04584116488695145 + }, + "val": { + "mean_centered_corr": -0.01837269775569439, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06368750333786011 + }, + "test": { + "mean_centered_corr": -0.019264845177531242, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08078610152006149 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 4, + 25, + 12, + 19, + 25, + 10 + ], + "head_selection_time": "0:00:00.103588" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..f3b5cf91b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24186190129822632, + "mae": 0.12565605342388153, + "r2": 0.6262778639793396, + "score": -0.24186190129822632 + }, + "val": { + "rmse": 0.23758917502158358, + "mae": 0.13536617159843445, + "r2": 0.6229451298713684, + "score": -0.23758917502158358 + }, + "test": { + "rmse": 0.23986609072189577, + "mae": 0.15381069481372833, + "r2": 0.60683274269104, + "score": -0.23986609072189577 + } + }, + "time": "0:02:02.109256", + "diversity": { + "train": { + "mean_centered_corr": -0.018595725297927856, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045762546360492706 + }, + "val": { + "mean_centered_corr": -0.01878608576953411, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06204522028565407 + }, + "test": { + "mean_centered_corr": -0.019662391394376755, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08173146098852158 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015281" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p005-evaluation/0.toml new file mode 100644 index 000000000..7a5f15280 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p005-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..6896eac74 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p005-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23179295341432377, + "mae": 0.12123288959264755, + "r2": 0.6567469835281372, + "score": -0.23179295341432377 + }, + "val": { + "rmse": 0.23474049679651998, + "mae": 0.12795619666576385, + "r2": 0.6319326162338257, + "score": -0.23474049679651998 + }, + "test": { + "rmse": 0.23271906309028628, + "mae": 0.1455690711736679, + "r2": 0.6299132704734802, + "score": -0.23271906309028628 + } + }, + "time": "0:02:02.109256", + "diversity": { + "train": { + "mean_centered_corr": -0.018595725297927856, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045762546360492706 + }, + "val": { + "mean_centered_corr": -0.01878608576953411, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06204522028565407 + }, + "test": { + "mean_centered_corr": -0.019662391394376755, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08173146098852158 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..607749a58 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,704 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23288190818323617, + "mae": 0.12223333865404129, + "r2": 0.6535142660140991, + "score": -0.23288190818323617 + }, + "val": { + "rmse": 0.23335307969400199, + "mae": 0.1274515837430954, + "r2": 0.6362706422805786, + "score": -0.23335307969400199 + }, + "test": { + "rmse": 0.23233217223360508, + "mae": 0.1452494114637375, + "r2": 0.6311427354812622, + "score": -0.23233217223360508 + } + }, + "time": "0:02:02.109256", + "diversity": { + "train": { + "mean_centered_corr": -0.018595725297927856, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045762546360492706 + }, + "val": { + "mean_centered_corr": -0.01878608576953411, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06204522028565407 + }, + "test": { + "mean_centered_corr": -0.019662391394376755, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08173146098852158 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 4, + 21, + 0, + 3, + 25, + 12, + 19, + 1, + 25, + 5, + 24 + ], + "head_selection_time": "0:00:00.175771" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..1977dabfd --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24408485538557934, + "mae": 0.12816353142261505, + "r2": 0.619376540184021, + "score": -0.24408485538557934 + }, + "val": { + "rmse": 0.23859870588230647, + "mae": 0.13501523435115814, + "r2": 0.6197340488433838, + "score": -0.23859870588230647 + }, + "test": { + "rmse": 0.23721550415536108, + "mae": 0.14897823333740234, + "r2": 0.6154739856719971, + "score": -0.23721550415536108 + } + }, + "time": "0:02:04.069929", + "diversity": { + "train": { + "mean_centered_corr": -0.019251784309744835, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04683033004403114 + }, + "val": { + "mean_centered_corr": -0.01828719675540924, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06498972326517105 + }, + "test": { + "mean_centered_corr": -0.01782698556780815, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08568309992551804 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015231" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p01-evaluation/0.toml new file mode 100644 index 000000000..7ba39f539 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p01-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..318083570 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p01-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23224626045326932, + "mae": 0.12201064825057983, + "r2": 0.6554031372070312, + "score": -0.23224626045326932 + }, + "val": { + "rmse": 0.23523273079765322, + "mae": 0.12723346054553986, + "r2": 0.6303874254226685, + "score": -0.23523273079765322 + }, + "test": { + "rmse": 0.231476592513396, + "mae": 0.1419006884098053, + "r2": 0.6338544487953186, + "score": -0.231476592513396 + } + }, + "time": "0:02:04.069929", + "diversity": { + "train": { + "mean_centered_corr": -0.019251784309744835, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04683033004403114 + }, + "val": { + "mean_centered_corr": -0.01828719675540924, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06498972326517105 + }, + "test": { + "mean_centered_corr": -0.01782698556780815, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08568309992551804 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..fea7976da --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,702 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2319716894083542, + "mae": 0.12287101149559021, + "r2": 0.6562174558639526, + "score": -0.2319716894083542 + }, + "val": { + "rmse": 0.23377952428751153, + "mae": 0.12833169102668762, + "r2": 0.6349400281906128, + "score": -0.23377952428751153 + }, + "test": { + "rmse": 0.23516218393385976, + "mae": 0.151117205619812, + "r2": 0.6221020221710205, + "score": -0.23516218393385976 + } + }, + "time": "0:02:04.069929", + "diversity": { + "train": { + "mean_centered_corr": -0.019251784309744835, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04683033004403114 + }, + "val": { + "mean_centered_corr": -0.01828719675540924, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06498972326517105 + }, + "test": { + "mean_centered_corr": -0.01782698556780815, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08568309992551804 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 4, + 0, + 21, + 28, + 25, + 12, + 17, + 19, + 18 + ], + "head_selection_time": "0:00:00.146054" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..10f8d933c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.22546390849445092, + "mae": 0.12138189375400543, + "r2": 0.67523592710495, + "score": -0.22546390849445092 + }, + "val": { + "rmse": 0.2384361951499822, + "mae": 0.13219110667705536, + "r2": 0.6202518939971924, + "score": -0.2384361951499822 + }, + "test": { + "rmse": 0.23890960422803306, + "mae": 0.14420710504055023, + "r2": 0.6099621057510376, + "score": -0.23890960422803306 + } + }, + "time": "0:02:16.435477", + "diversity": { + "train": { + "mean_centered_corr": -0.017872996628284454, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04516569897532463 + }, + "val": { + "mean_centered_corr": -0.01949959062039852, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06368565559387207 + }, + "test": { + "mean_centered_corr": -0.016032706946134567, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08689064532518387 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 8 + ], + "head_selection_time": "0:00:00.015277" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p001-evaluation/0.toml new file mode 100644 index 000000000..39a3eb794 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p001-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.95 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..91783d0ab --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p001-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.21667307678393335, + "mae": 0.1152627021074295, + "r2": 0.7000673413276672, + "score": -0.21667307678393335 + }, + "val": { + "rmse": 0.2352393029012561, + "mae": 0.12733106315135956, + "r2": 0.6303667426109314, + "score": -0.2352393029012561 + }, + "test": { + "rmse": 0.23369680670218232, + "mae": 0.14800961315631866, + "r2": 0.6267969608306885, + "score": -0.23369680670218232 + } + }, + "time": "0:02:16.435477", + "diversity": { + "train": { + "mean_centered_corr": -0.017872996628284454, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04516569897532463 + }, + "val": { + "mean_centered_corr": -0.01949959062039852, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06368565559387207 + }, + "test": { + "mean_centered_corr": -0.016032706946134567, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08689064532518387 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..cf3e69226 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,699 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2179749245264706, + "mae": 0.11718267947435379, + "r2": 0.696452260017395, + "score": -0.2179749245264706 + }, + "val": { + "rmse": 0.2338562469701412, + "mae": 0.1282421499490738, + "r2": 0.6347004175186157, + "score": -0.2338562469701412 + }, + "test": { + "rmse": 0.23357564989983745, + "mae": 0.1465408205986023, + "r2": 0.6271837949752808, + "score": -0.23357564989983745 + } + }, + "time": "0:02:16.435477", + "diversity": { + "train": { + "mean_centered_corr": -0.017872996628284454, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04516569897532463 + }, + "val": { + "mean_centered_corr": -0.01949959062039852, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06368565559387207 + }, + "test": { + "mean_centered_corr": -0.016032706946134567, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08689064532518387 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 8, + 0, + 4, + 19, + 21, + 25 + ], + "head_selection_time": "0:00:00.101719" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..5190a126a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24092140272621057, + "mae": 0.12480571866035461, + "r2": 0.6291787028312683, + "score": -0.24092140272621057 + }, + "val": { + "rmse": 0.2371704208371665, + "mae": 0.12888024747371674, + "r2": 0.6242730617523193, + "score": -0.2371704208371665 + }, + "test": { + "rmse": 0.23451416334598613, + "mae": 0.14349374175071716, + "r2": 0.624181866645813, + "score": -0.23451416334598613 + } + }, + "time": "0:02:05.620079", + "diversity": { + "train": { + "mean_centered_corr": -0.018756726756691933, + "mean_pairwise_disagreement": NaN, + "member_std": 0.044779445976018906 + }, + "val": { + "mean_centered_corr": -0.019936703145503998, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0639626681804657 + }, + "test": { + "mean_centered_corr": -0.01880541257560253, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07910130172967911 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015272" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p005-evaluation/0.toml new file mode 100644 index 000000000..7db59b766 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p005-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.95 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..c18abb04b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p005-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23337935525306813, + "mae": 0.12134772539138794, + "r2": 0.6520324349403381, + "score": -0.23337935525306813 + }, + "val": { + "rmse": 0.23530708779168627, + "mae": 0.12533041834831238, + "r2": 0.6301536560058594, + "score": -0.23530708779168627 + }, + "test": { + "rmse": 0.23121293389473055, + "mae": 0.1421053111553192, + "r2": 0.6346880793571472, + "score": -0.23121293389473055 + } + }, + "time": "0:02:05.620079", + "diversity": { + "train": { + "mean_centered_corr": -0.018756726756691933, + "mean_pairwise_disagreement": NaN, + "member_std": 0.044779445976018906 + }, + "val": { + "mean_centered_corr": -0.019936703145503998, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0639626681804657 + }, + "test": { + "mean_centered_corr": -0.01880541257560253, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07910130172967911 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..e4b9580d8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,698 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23514631827029459, + "mae": 0.12384053319692612, + "r2": 0.6467434167861938, + "score": -0.23514631827029459 + }, + "val": { + "rmse": 0.23311429304688983, + "mae": 0.12682734429836273, + "r2": 0.6370146870613098, + "score": -0.23311429304688983 + }, + "test": { + "rmse": 0.23653039079394922, + "mae": 0.15279927849769592, + "r2": 0.6176918745040894, + "score": -0.23653039079394922 + } + }, + "time": "0:02:05.620079", + "diversity": { + "train": { + "mean_centered_corr": -0.018756726756691933, + "mean_pairwise_disagreement": NaN, + "member_std": 0.044779445976018906 + }, + "val": { + "mean_centered_corr": -0.019936703145503998, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0639626681804657 + }, + "test": { + "mean_centered_corr": -0.01880541257560253, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07910130172967911 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 4, + 0, + 19, + 1, + 25 + ], + "head_selection_time": "0:00:00.087358" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..6628130af --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24083051915130507, + "mae": 0.1239011138677597, + "r2": 0.6294584274291992, + "score": -0.24083051915130507 + }, + "val": { + "rmse": 0.23633714926648194, + "mae": 0.13169743120670319, + "r2": 0.6269086003303528, + "score": -0.23633714926648194 + }, + "test": { + "rmse": 0.23801691214143877, + "mae": 0.1493917554616928, + "r2": 0.6128714084625244, + "score": -0.23801691214143877 + } + }, + "time": "0:02:05.602883", + "diversity": { + "train": { + "mean_centered_corr": -0.019140595570206642, + "mean_pairwise_disagreement": NaN, + "member_std": 0.044947654008865356 + }, + "val": { + "mean_centered_corr": -0.018659241497516632, + "mean_pairwise_disagreement": NaN, + "member_std": 0.060750968754291534 + }, + "test": { + "mean_centered_corr": -0.015921765938401222, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07888734340667725 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015263" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p01-evaluation/0.toml new file mode 100644 index 000000000..c97d7875f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p01-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.95 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..c7d765f3a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p01-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23298860424389348, + "mae": 0.12124313414096832, + "r2": 0.6531966924667358, + "score": -0.23298860424389348 + }, + "val": { + "rmse": 0.234867262459557, + "mae": 0.1260262280702591, + "r2": 0.6315350532531738, + "score": -0.234867262459557 + }, + "test": { + "rmse": 0.23225197071547168, + "mae": 0.1443255990743637, + "r2": 0.6313973665237427, + "score": -0.23225197071547168 + } + }, + "time": "0:02:05.602883", + "diversity": { + "train": { + "mean_centered_corr": -0.019140595570206642, + "mean_pairwise_disagreement": NaN, + "member_std": 0.044947654008865356 + }, + "val": { + "mean_centered_corr": -0.018659241497516632, + "mean_pairwise_disagreement": NaN, + "member_std": 0.060750968754291534 + }, + "test": { + "mean_centered_corr": -0.015921765938401222, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07888734340667725 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..8d0d6d073 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,697 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23454354107218955, + "mae": 0.12458474934101105, + "r2": 0.6485521793365479, + "score": -0.23454354107218955 + }, + "val": { + "rmse": 0.2329376812231261, + "mae": 0.1272622048854828, + "r2": 0.637564480304718, + "score": -0.2329376812231261 + }, + "test": { + "rmse": 0.23722725849121748, + "mae": 0.15431377291679382, + "r2": 0.6154358386993408, + "score": -0.23722725849121748 + } + }, + "time": "0:02:05.602883", + "diversity": { + "train": { + "mean_centered_corr": -0.019140595570206642, + "mean_pairwise_disagreement": NaN, + "member_std": 0.044947654008865356 + }, + "val": { + "mean_centered_corr": -0.018659241497516632, + "mean_pairwise_disagreement": NaN, + "member_std": 0.060750968754291534 + }, + "test": { + "mean_centered_corr": -0.015921765938401222, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07888734340667725 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 4, + 25, + 21, + 19 + ], + "head_selection_time": "0:00:00.073012" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..4a8a67898 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2246712226303735, + "mae": 0.12456091493368149, + "r2": 0.6775155067443848, + "score": -0.2246712226303735 + }, + "val": { + "rmse": 0.23738497878870984, + "mae": 0.1353979855775833, + "r2": 0.6235929727554321, + "score": -0.23738497878870984 + }, + "test": { + "rmse": 0.2429803744292108, + "mae": 0.157321497797966, + "r2": 0.5965571403503418, + "score": -0.2429803744292108 + } + }, + "time": "0:02:14.535269", + "diversity": { + "train": { + "mean_centered_corr": -0.01842476613819599, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04623854160308838 + }, + "val": { + "mean_centered_corr": -0.018401212990283966, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06464710086584091 + }, + "test": { + "mean_centered_corr": -0.016835635527968407, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09058339893817902 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015185" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p001-evaluation/0.toml new file mode 100644 index 000000000..06fea7ad1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p001-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..85bbd92dc --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p001-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.21620215214607402, + "mae": 0.11695478111505508, + "r2": 0.7013696432113647, + "score": -0.21620215214607402 + }, + "val": { + "rmse": 0.23505533340532211, + "mae": 0.12693415582180023, + "r2": 0.6309446692466736, + "score": -0.23505533340532211 + }, + "test": { + "rmse": 0.2334740169208309, + "mae": 0.1462247669696808, + "r2": 0.6275081634521484, + "score": -0.2334740169208309 + } + }, + "time": "0:02:14.535269", + "diversity": { + "train": { + "mean_centered_corr": -0.01842476613819599, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04623854160308838 + }, + "val": { + "mean_centered_corr": -0.018401212990283966, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06464710086584091 + }, + "test": { + "mean_centered_corr": -0.016835635527968407, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09058339893817902 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..6fdb8ee65 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,699 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.22155602825004872, + "mae": 0.1207282543182373, + "r2": 0.6863964200019836, + "score": -0.22155602825004872 + }, + "val": { + "rmse": 0.23371274685689286, + "mae": 0.1293759047985077, + "r2": 0.6351485848426819, + "score": -0.23371274685689286 + }, + "test": { + "rmse": 0.23853136443639264, + "mae": 0.157401904463768, + "r2": 0.6111961603164673, + "score": -0.23853136443639264 + } + }, + "time": "0:02:14.535269", + "diversity": { + "train": { + "mean_centered_corr": -0.01842476613819599, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04623854160308838 + }, + "val": { + "mean_centered_corr": -0.018401212990283966, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06464710086584091 + }, + "test": { + "mean_centered_corr": -0.016835635527968407, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09058339893817902 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 4, + 25, + 28, + 22, + 6, + 1 + ], + "head_selection_time": "0:00:00.101402" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..f313e0843 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23852410213626124, + "mae": 0.12188179790973663, + "r2": 0.6365216970443726, + "score": -0.23852410213626124 + }, + "val": { + "rmse": 0.23674080812018197, + "mae": 0.12981633841991425, + "r2": 0.6256330013275146, + "score": -0.23674080812018197 + }, + "test": { + "rmse": 0.24022462096222602, + "mae": 0.15470069646835327, + "r2": 0.6056565642356873, + "score": -0.24022462096222602 + } + }, + "time": "0:02:06.448494", + "diversity": { + "train": { + "mean_centered_corr": -0.018909571692347527, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04599001631140709 + }, + "val": { + "mean_centered_corr": -0.018043946474790573, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06483229994773865 + }, + "test": { + "mean_centered_corr": -0.018734198063611984, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08063172549009323 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015132" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p005-evaluation/0.toml new file mode 100644 index 000000000..a460d2818 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p005-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..5ef08a624 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p005-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2299791404863945, + "mae": 0.12113875150680542, + "r2": 0.6620979905128479, + "score": -0.2299791404863945 + }, + "val": { + "rmse": 0.23498608081258596, + "mae": 0.12758168578147888, + "r2": 0.6311621069908142, + "score": -0.23498608081258596 + }, + "test": { + "rmse": 0.23378104608060551, + "mae": 0.14829501509666443, + "r2": 0.6265279054641724, + "score": -0.23378104608060551 + } + }, + "time": "0:02:06.448494", + "diversity": { + "train": { + "mean_centered_corr": -0.018909571692347527, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04599001631140709 + }, + "val": { + "mean_centered_corr": -0.018043946474790573, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06483229994773865 + }, + "test": { + "mean_centered_corr": -0.018734198063611984, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08063172549009323 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..2e44f4c73 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,700 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23242986489034317, + "mae": 0.12031129002571106, + "r2": 0.6548580527305603, + "score": -0.23242986489034317 + }, + "val": { + "rmse": 0.2333506132153353, + "mae": 0.1259656697511673, + "r2": 0.6362783908843994, + "score": -0.2333506132153353 + }, + "test": { + "rmse": 0.23993873237607374, + "mae": 0.15797272324562073, + "r2": 0.6065946221351624, + "score": -0.23993873237607374 + } + }, + "time": "0:02:06.448494", + "diversity": { + "train": { + "mean_centered_corr": -0.018909571692347527, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04599001631140709 + }, + "val": { + "mean_centered_corr": -0.018043946474790573, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06483229994773865 + }, + "test": { + "mean_centered_corr": -0.018734198063611984, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08063172549009323 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 4, + 0, + 21, + 25, + 10, + 20, + 1 + ], + "head_selection_time": "0:00:00.116624" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..570f97feb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.22891393817840822, + "mae": 0.1278449147939682, + "r2": 0.6652208566665649, + "score": -0.22891393817840822 + }, + "val": { + "rmse": 0.24086258324668144, + "mae": 0.14345626533031464, + "r2": 0.6124837398529053, + "score": -0.24086258324668144 + }, + "test": { + "rmse": 0.2704202668685561, + "mae": 0.193245068192482, + "r2": 0.5002899765968323, + "score": -0.2704202668685561 + } + }, + "time": "0:02:18.381520", + "diversity": { + "train": { + "mean_centered_corr": -0.018370768055319786, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04778972268104553 + }, + "val": { + "mean_centered_corr": -0.01997104473412037, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06924484670162201 + }, + "test": { + "mean_centered_corr": -0.018939772620797157, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08999743312597275 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 25 + ], + "head_selection_time": "0:00:00.015255" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p01-evaluation/0.toml new file mode 100644 index 000000000..b006acb29 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p01-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..1c243c72c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p01-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2169704086402944, + "mae": 0.11638687551021576, + "r2": 0.6992435455322266, + "score": -0.2169704086402944 + }, + "val": { + "rmse": 0.23478007280066537, + "mae": 0.12811791896820068, + "r2": 0.6318085193634033, + "score": -0.23478007280066537 + }, + "test": { + "rmse": 0.2351072792694515, + "mae": 0.14944614470005035, + "r2": 0.6222784519195557, + "score": -0.2351072792694515 + } + }, + "time": "0:02:18.381520", + "diversity": { + "train": { + "mean_centered_corr": -0.018370768055319786, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04778972268104553 + }, + "val": { + "mean_centered_corr": -0.01997104473412037, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06924484670162201 + }, + "test": { + "mean_centered_corr": -0.018939772620797157, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08999743312597275 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..853f683ed --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,707 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.21863364633384474, + "mae": 0.11706539988517761, + "r2": 0.6946148872375488, + "score": -0.21863364633384474 + }, + "val": { + "rmse": 0.23354353444721934, + "mae": 0.12788079679012299, + "r2": 0.6356767416000366, + "score": -0.23354353444721934 + }, + "test": { + "rmse": 0.2390892602865205, + "mae": 0.15497441589832306, + "r2": 0.6093752980232239, + "score": -0.2390892602865205 + } + }, + "time": "0:02:18.381520", + "diversity": { + "train": { + "mean_centered_corr": -0.018370768055319786, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04778972268104553 + }, + "val": { + "mean_centered_corr": -0.01997104473412037, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06924484670162201 + }, + "test": { + "mean_centered_corr": -0.018939772620797157, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08999743312597275 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 25, + 7, + 10, + 22, + 21, + 14, + 31, + 12, + 6, + 10, + 22, + 27, + 19, + 10 + ], + "head_selection_time": "0:00:00.219853" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..35188f4ee --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.22951726573518907, + "mae": 0.12178733944892883, + "r2": 0.6634538769721985, + "score": -0.22951726573518907 + }, + "val": { + "rmse": 0.237046057654127, + "mae": 0.13444463908672333, + "r2": 0.6246670484542847, + "score": -0.237046057654127 + }, + "test": { + "rmse": 0.2530936074511429, + "mae": 0.17497020959854126, + "r2": 0.5622743964195251, + "score": -0.2530936074511429 + } + }, + "time": "0:02:19.714907", + "diversity": { + "train": { + "mean_centered_corr": -0.018565235659480095, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04782221093773842 + }, + "val": { + "mean_centered_corr": -0.01974944956600666, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06836902350187302 + }, + "test": { + "mean_centered_corr": -0.01930362544953823, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09177923202514648 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 25 + ], + "head_selection_time": "0:00:00.015486" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p001-evaluation/0.toml new file mode 100644 index 000000000..9206c4214 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p001-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..1d06e85e1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p001-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.21750608206883032, + "mae": 0.11355320364236832, + "r2": 0.6977567076683044, + "score": -0.21750608206883032 + }, + "val": { + "rmse": 0.2350854992177386, + "mae": 0.12681186199188232, + "r2": 0.6308499574661255, + "score": -0.2350854992177386 + }, + "test": { + "rmse": 0.23319390245153368, + "mae": 0.14503715932369232, + "r2": 0.628401517868042, + "score": -0.23319390245153368 + } + }, + "time": "0:02:19.714907", + "diversity": { + "train": { + "mean_centered_corr": -0.018565235659480095, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04782221093773842 + }, + "val": { + "mean_centered_corr": -0.01974944956600666, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06836902350187302 + }, + "test": { + "mean_centered_corr": -0.01930362544953823, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09177923202514648 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..95d9be15f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,701 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.22152409575796225, + "mae": 0.1159883365035057, + "r2": 0.6864867806434631, + "score": -0.22152409575796225 + }, + "val": { + "rmse": 0.23381808394805512, + "mae": 0.12612800300121307, + "r2": 0.6348196268081665, + "score": -0.23381808394805512 + }, + "test": { + "rmse": 0.230425386066968, + "mae": 0.13822335004806519, + "r2": 0.6371724605560303, + "score": -0.230425386066968 + } + }, + "time": "0:02:19.714907", + "diversity": { + "train": { + "mean_centered_corr": -0.018565235659480095, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04782221093773842 + }, + "val": { + "mean_centered_corr": -0.01974944956600666, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06836902350187302 + }, + "test": { + "mean_centered_corr": -0.01930362544953823, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09177923202514648 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 25, + 9, + 5, + 0, + 19, + 22, + 29, + 3 + ], + "head_selection_time": "0:00:00.131585" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..3aada2516 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2429541865580833, + "mae": 0.1267397403717041, + "r2": 0.6228946447372437, + "score": -0.2429541865580833 + }, + "val": { + "rmse": 0.23844788931164226, + "mae": 0.13394595682621002, + "r2": 0.6202146410942078, + "score": -0.23844788931164226 + }, + "test": { + "rmse": 0.239992780080121, + "mae": 0.152608722448349, + "r2": 0.6064173579216003, + "score": -0.239992780080121 + } + }, + "time": "0:02:02.867107", + "diversity": { + "train": { + "mean_centered_corr": -0.019329514354467392, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04684953764081001 + }, + "val": { + "mean_centered_corr": -0.01879928633570671, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0651373490691185 + }, + "test": { + "mean_centered_corr": -0.02050378918647766, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08481118083000183 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015382" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p005-evaluation/0.toml new file mode 100644 index 000000000..8d4da8b93 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p005-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..3b3aa2a82 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p005-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23163036400903525, + "mae": 0.12018397450447083, + "r2": 0.6572283506393433, + "score": -0.23163036400903525 + }, + "val": { + "rmse": 0.23460196790457225, + "mae": 0.1260966658592224, + "r2": 0.632366955280304, + "score": -0.23460196790457225 + }, + "test": { + "rmse": 0.23160625447220415, + "mae": 0.1424369513988495, + "r2": 0.6334441900253296, + "score": -0.23160625447220415 + } + }, + "time": "0:02:02.867107", + "diversity": { + "train": { + "mean_centered_corr": -0.019329514354467392, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04684953764081001 + }, + "val": { + "mean_centered_corr": -0.01879928633570671, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0651373490691185 + }, + "test": { + "mean_centered_corr": -0.02050378918647766, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08481118083000183 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..bbf779053 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,700 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23151798153687342, + "mae": 0.12077950686216354, + "r2": 0.6575609445571899, + "score": -0.23151798153687342 + }, + "val": { + "rmse": 0.23278091651666263, + "mae": 0.12862801551818848, + "r2": 0.6380521655082703, + "score": -0.23278091651666263 + }, + "test": { + "rmse": 0.23692641701684622, + "mae": 0.15434525907039642, + "r2": 0.6164106130599976, + "score": -0.23692641701684622 + } + }, + "time": "0:02:02.867107", + "diversity": { + "train": { + "mean_centered_corr": -0.019329514354467392, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04684953764081001 + }, + "val": { + "mean_centered_corr": -0.01879928633570671, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0651373490691185 + }, + "test": { + "mean_centered_corr": -0.02050378918647766, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08481118083000183 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 4, + 25, + 12, + 18, + 29, + 25, + 19 + ], + "head_selection_time": "0:00:00.116591" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..a9c72a6c4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24471092752622822, + "mae": 0.12746074795722961, + "r2": 0.6174214482307434, + "score": -0.24471092752622822 + }, + "val": { + "rmse": 0.23860736324692997, + "mae": 0.13630448281764984, + "r2": 0.6197065114974976, + "score": -0.23860736324692997 + }, + "test": { + "rmse": 0.24541511515454548, + "mae": 0.16491781175136566, + "r2": 0.5884314179420471, + "score": -0.24541511515454548 + } + }, + "time": "0:02:05.625831", + "diversity": { + "train": { + "mean_centered_corr": -0.019371401518583298, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046141691505908966 + }, + "val": { + "mean_centered_corr": -0.01898176036775112, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06393436342477798 + }, + "test": { + "mean_centered_corr": -0.019255701452493668, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08550143241882324 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015058" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p01-evaluation/0.toml new file mode 100644 index 000000000..55eb6a51e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p01-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..d16c1f969 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p01-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23234667483081278, + "mae": 0.12070997804403305, + "r2": 0.6551050543785095, + "score": -0.23234667483081278 + }, + "val": { + "rmse": 0.23525231195767085, + "mae": 0.1279769092798233, + "r2": 0.6303259134292603, + "score": -0.23525231195767085 + }, + "test": { + "rmse": 0.23376059267196012, + "mae": 0.14717192947864532, + "r2": 0.6265932321548462, + "score": -0.23376059267196012 + } + }, + "time": "0:02:05.625831", + "diversity": { + "train": { + "mean_centered_corr": -0.019371401518583298, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046141691505908966 + }, + "val": { + "mean_centered_corr": -0.01898176036775112, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06393436342477798 + }, + "test": { + "mean_centered_corr": -0.019255701452493668, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08550143241882324 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..96be49ce3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,704 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23244703781733334, + "mae": 0.12160777300596237, + "r2": 0.6548070907592773, + "score": -0.23244703781733334 + }, + "val": { + "rmse": 0.23326554764868923, + "mae": 0.1277935653924942, + "r2": 0.6365435123443604, + "score": -0.23326554764868923 + }, + "test": { + "rmse": 0.24138003765619687, + "mae": 0.16120430827140808, + "r2": 0.6018540859222412, + "score": -0.24138003765619687 + } + }, + "time": "0:02:05.625831", + "diversity": { + "train": { + "mean_centered_corr": -0.019371401518583298, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046141691505908966 + }, + "val": { + "mean_centered_corr": -0.01898176036775112, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06393436342477798 + }, + "test": { + "mean_centered_corr": -0.019255701452493668, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08550143241882324 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 4, + 25, + 12, + 18, + 20, + 21, + 25, + 12, + 25, + 19, + 16 + ], + "head_selection_time": "0:00:00.173565" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..9065a10a8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23908905773123074, + "mae": 0.13352937996387482, + "r2": 0.6347978711128235, + "score": -0.23908905773123074 + }, + "val": { + "rmse": 0.23745188445279078, + "mae": 0.13348934054374695, + "r2": 0.6233807802200317, + "score": -0.23745188445279078 + }, + "test": { + "rmse": 0.23531645200331283, + "mae": 0.14286726713180542, + "r2": 0.6216060519218445, + "score": -0.23531645200331283 + } + }, + "time": "0:02:05.428840", + "diversity": { + "train": { + "mean_centered_corr": -0.01883065141737461, + "mean_pairwise_disagreement": NaN, + "member_std": 0.043418336659669876 + }, + "val": { + "mean_centered_corr": -0.019489828497171402, + "mean_pairwise_disagreement": NaN, + "member_std": 0.058777473866939545 + }, + "test": { + "mean_centered_corr": -0.017089199274778366, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07344797253608704 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 0 + ], + "head_selection_time": "0:00:00.015304" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p001-evaluation/0.toml new file mode 100644 index 000000000..a47046c92 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p001-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.95 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..484eee180 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p001-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2330807634805131, + "mae": 0.12179981172084808, + "r2": 0.6529222726821899, + "score": -0.2330807634805131 + }, + "val": { + "rmse": 0.23558367762733304, + "mae": 0.12585210800170898, + "r2": 0.6292837262153625, + "score": -0.23558367762733304 + }, + "test": { + "rmse": 0.2306278293973631, + "mae": 0.14087699353694916, + "r2": 0.6365346312522888, + "score": -0.2306278293973631 + } + }, + "time": "0:02:05.428840", + "diversity": { + "train": { + "mean_centered_corr": -0.01883065141737461, + "mean_pairwise_disagreement": NaN, + "member_std": 0.043418336659669876 + }, + "val": { + "mean_centered_corr": -0.019489828497171402, + "mean_pairwise_disagreement": NaN, + "member_std": 0.058777473866939545 + }, + "test": { + "mean_centered_corr": -0.017089199274778366, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07344797253608704 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..1d4057f55 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,703 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23343546426030956, + "mae": 0.12315606325864792, + "r2": 0.6518651247024536, + "score": -0.23343546426030956 + }, + "val": { + "rmse": 0.2330749056995843, + "mae": 0.12734414637088776, + "r2": 0.6371372938156128, + "score": -0.2330749056995843 + }, + "test": { + "rmse": 0.23692981324824752, + "mae": 0.1541774570941925, + "r2": 0.6163996458053589, + "score": -0.23692981324824752 + } + }, + "time": "0:02:05.428840", + "diversity": { + "train": { + "mean_centered_corr": -0.01883065141737461, + "mean_pairwise_disagreement": NaN, + "member_std": 0.043418336659669876 + }, + "val": { + "mean_centered_corr": -0.019489828497171402, + "mean_pairwise_disagreement": NaN, + "member_std": 0.058777473866939545 + }, + "test": { + "mean_centered_corr": -0.017089199274778366, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07344797253608704 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 0, + 25, + 21, + 25, + 4, + 19, + 6, + 1, + 25, + 22 + ], + "head_selection_time": "0:00:00.158945" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..cc7e36898 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23844750654589067, + "mae": 0.12963111698627472, + "r2": 0.6367551684379578, + "score": -0.23844750654589067 + }, + "val": { + "rmse": 0.23707190033649148, + "mae": 0.1328258216381073, + "r2": 0.6245851516723633, + "score": -0.23707190033649148 + }, + "test": { + "rmse": 0.2397359546762736, + "mae": 0.1563534289598465, + "r2": 0.6072592735290527, + "score": -0.2397359546762736 + } + }, + "time": "0:02:03.794551", + "diversity": { + "train": { + "mean_centered_corr": -0.018837179988622665, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04427312687039375 + }, + "val": { + "mean_centered_corr": -0.020433422178030014, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05948192626237869 + }, + "test": { + "mean_centered_corr": -0.018294233828783035, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07480351626873016 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 0 + ], + "head_selection_time": "0:00:00.015171" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p005-evaluation/0.toml new file mode 100644 index 000000000..979ff2558 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p005-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.95 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..27d9cb616 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p005-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23271073895226718, + "mae": 0.12118933349847794, + "r2": 0.6540234088897705, + "score": -0.23271073895226718 + }, + "val": { + "rmse": 0.23476767229656906, + "mae": 0.12631559371948242, + "r2": 0.6318474411964417, + "score": -0.23476767229656906 + }, + "test": { + "rmse": 0.23170820829575262, + "mae": 0.14362695813179016, + "r2": 0.6331213712692261, + "score": -0.23170820829575262 + } + }, + "time": "0:02:03.794551", + "diversity": { + "train": { + "mean_centered_corr": -0.018837179988622665, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04427312687039375 + }, + "val": { + "mean_centered_corr": -0.020433422178030014, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05948192626237869 + }, + "test": { + "mean_centered_corr": -0.018294233828783035, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07480351626873016 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..24d1febd6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,700 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23438370211658663, + "mae": 0.12246029078960419, + "r2": 0.6490310430526733, + "score": -0.23438370211658663 + }, + "val": { + "rmse": 0.23225748836355237, + "mae": 0.12540538609027863, + "r2": 0.6396780610084534, + "score": -0.23225748836355237 + }, + "test": { + "rmse": 0.2373654401585729, + "mae": 0.1536472588777542, + "r2": 0.6149877309799194, + "score": -0.2373654401585729 + } + }, + "time": "0:02:03.794551", + "diversity": { + "train": { + "mean_centered_corr": -0.018837179988622665, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04427312687039375 + }, + "val": { + "mean_centered_corr": -0.020433422178030014, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05948192626237869 + }, + "test": { + "mean_centered_corr": -0.018294233828783035, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07480351626873016 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 0, + 4, + 25, + 21, + 19, + 25, + 21 + ], + "head_selection_time": "0:00:00.130562" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..2415d6d35 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2323950263151425, + "mae": 0.12666819989681244, + "r2": 0.6549615263938904, + "score": -0.2323950263151425 + }, + "val": { + "rmse": 0.2386455487030765, + "mae": 0.14065667986869812, + "r2": 0.6195847392082214, + "score": -0.2386455487030765 + }, + "test": { + "rmse": 0.27701433524933666, + "mae": 0.19954755902290344, + "r2": 0.4756224751472473, + "score": -0.27701433524933666 + } + }, + "time": "0:02:18.086910", + "diversity": { + "train": { + "mean_centered_corr": -0.01866568624973297, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04555417224764824 + }, + "val": { + "mean_centered_corr": -0.01869249902665615, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06710393726825714 + }, + "test": { + "mean_centered_corr": -0.01764187403023243, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08898688107728958 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 25 + ], + "head_selection_time": "0:00:00.015682" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p01-evaluation/0.toml new file mode 100644 index 000000000..5c0b05d96 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p01-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.95 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..5fce97243 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p01-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.21881660230888528, + "mae": 0.11528468877077103, + "r2": 0.6941035985946655, + "score": -0.21881660230888528 + }, + "val": { + "rmse": 0.2351196777064841, + "mae": 0.12626346945762634, + "r2": 0.630742609500885, + "score": -0.2351196777064841 + }, + "test": { + "rmse": 0.23285365668962787, + "mae": 0.14447025954723358, + "r2": 0.6294850707054138, + "score": -0.23285365668962787 + } + }, + "time": "0:02:18.086910", + "diversity": { + "train": { + "mean_centered_corr": -0.01866568624973297, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04555417224764824 + }, + "val": { + "mean_centered_corr": -0.01869249902665615, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06710393726825714 + }, + "test": { + "mean_centered_corr": -0.01764187403023243, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08898688107728958 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..48a5a1aa2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,698 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.22086028183547482, + "mae": 0.11741601675748825, + "r2": 0.6883629560470581, + "score": -0.22086028183547482 + }, + "val": { + "rmse": 0.23375630575487258, + "mae": 0.12780286371707916, + "r2": 0.6350125074386597, + "score": -0.23375630575487258 + }, + "test": { + "rmse": 0.23772689559358076, + "mae": 0.15195266902446747, + "r2": 0.6138142347335815, + "score": -0.23772689559358076 + } + }, + "time": "0:02:18.086910", + "diversity": { + "train": { + "mean_centered_corr": -0.01866568624973297, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04555417224764824 + }, + "val": { + "mean_centered_corr": -0.01869249902665615, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06710393726825714 + }, + "test": { + "mean_centered_corr": -0.01764187403023243, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08898688107728958 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 25, + 8, + 21, + 7, + 12 + ], + "head_selection_time": "0:00:00.090438" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..4bc6f4ede --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.23131872489766006, + "mae": 0.1263350546360016, + "r2": 0.6581500768661499, + "score": -0.23131872489766006 + }, + "val": { + "rmse": 0.23771184363825437, + "mae": 0.13543455302715302, + "r2": 0.622555673122406, + "score": -0.23771184363825437 + }, + "test": { + "rmse": 0.2596801100779992, + "mae": 0.17902591824531555, + "r2": 0.53919517993927, + "score": -0.2596801100779992 + } + }, + "time": "0:02:19.164877", + "diversity": { + "train": { + "mean_centered_corr": -0.01825721003115177, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045883432030677795 + }, + "val": { + "mean_centered_corr": -0.019998809322714806, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0684179738163948 + }, + "test": { + "mean_centered_corr": -0.019780565053224564, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08797822892665863 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 25 + ], + "head_selection_time": "0:00:00.015296" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p001-evaluation/0.toml new file mode 100644 index 000000000..996f958aa --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p001-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..822ae80ed --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p001-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.21707002083369387, + "mae": 0.11528186500072479, + "r2": 0.6989673376083374, + "score": -0.21707002083369387 + }, + "val": { + "rmse": 0.23556640917246102, + "mae": 0.12705539166927338, + "r2": 0.6293380260467529, + "score": -0.23556640917246102 + }, + "test": { + "rmse": 0.23394963288342588, + "mae": 0.14626435935497284, + "r2": 0.6259890794754028, + "score": -0.23394963288342588 + } + }, + "time": "0:02:19.164877", + "diversity": { + "train": { + "mean_centered_corr": -0.01825721003115177, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045883432030677795 + }, + "val": { + "mean_centered_corr": -0.019998809322714806, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0684179738163948 + }, + "test": { + "mean_centered_corr": -0.019780565053224564, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08797822892665863 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..c55531ba4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,702 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.22058343595772106, + "mae": 0.11804859340190887, + "r2": 0.689143717288971, + "score": -0.22058343595772106 + }, + "val": { + "rmse": 0.2337060760344375, + "mae": 0.12719006836414337, + "r2": 0.6351693868637085, + "score": -0.2337060760344375 + }, + "test": { + "rmse": 0.23617809041406845, + "mae": 0.14892512559890747, + "r2": 0.6188299059867859, + "score": -0.23617809041406845 + } + }, + "time": "0:02:19.164877", + "diversity": { + "train": { + "mean_centered_corr": -0.01825721003115177, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045883432030677795 + }, + "val": { + "mean_centered_corr": -0.019998809322714806, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0684179738163948 + }, + "test": { + "mean_centered_corr": -0.019780565053224564, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08797822892665863 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 25, + 4, + 19, + 0, + 16, + 22, + 10, + 20, + 4 + ], + "head_selection_time": "0:00:00.146000" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..bda9dcf87 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23600635218658622, + "mae": 0.12171389162540436, + "r2": 0.6441546678543091, + "score": -0.23600635218658622 + }, + "val": { + "rmse": 0.23788508328903052, + "mae": 0.13621799647808075, + "r2": 0.6220053434371948, + "score": -0.23788508328903052 + }, + "test": { + "rmse": 0.2505198973693071, + "mae": 0.171568363904953, + "r2": 0.5711315870285034, + "score": -0.2505198973693071 + } + }, + "time": "0:02:01.587396", + "diversity": { + "train": { + "mean_centered_corr": -0.01862383633852005, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04512369632720947 + }, + "val": { + "mean_centered_corr": -0.01866878941655159, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06477370113134384 + }, + "test": { + "mean_centered_corr": -0.019034983590245247, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08236686140298843 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015370" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p005-evaluation/0.toml new file mode 100644 index 000000000..c1d2bcf74 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p005-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..b140bc931 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p005-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23003808694853833, + "mae": 0.12140953540802002, + "r2": 0.6619247198104858, + "score": -0.23003808694853833 + }, + "val": { + "rmse": 0.23502662197192856, + "mae": 0.12718608975410461, + "r2": 0.6310348510742188, + "score": -0.23502662197192856 + }, + "test": { + "rmse": 0.23487478850600307, + "mae": 0.149356409907341, + "r2": 0.623025119304657, + "score": -0.23487478850600307 + } + }, + "time": "0:02:01.587396", + "diversity": { + "train": { + "mean_centered_corr": -0.01862383633852005, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04512369632720947 + }, + "val": { + "mean_centered_corr": -0.01866878941655159, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06477370113134384 + }, + "test": { + "mean_centered_corr": -0.019034983590245247, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08236686140298843 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..19dc64a59 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,701 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23224673364044418, + "mae": 0.12295585125684738, + "r2": 0.6554017066955566, + "score": -0.23224673364044418 + }, + "val": { + "rmse": 0.23340701642091277, + "mae": 0.12782487273216248, + "r2": 0.6361024975776672, + "score": -0.23340701642091277 + }, + "test": { + "rmse": 0.24153220831875505, + "mae": 0.16074170172214508, + "r2": 0.6013518571853638, + "score": -0.24153220831875505 + } + }, + "time": "0:02:01.587396", + "diversity": { + "train": { + "mean_centered_corr": -0.01862383633852005, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04512369632720947 + }, + "val": { + "mean_centered_corr": -0.01866878941655159, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06477370113134384 + }, + "test": { + "mean_centered_corr": -0.019034983590245247, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08236686140298843 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 4, + 21, + 25, + 10, + 19, + 21, + 0, + 16 + ], + "head_selection_time": "0:00:00.133352" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..62af2190c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2242404395588436, + "mae": 0.13221053779125214, + "r2": 0.6787509918212891, + "score": -0.2242404395588436 + }, + "val": { + "rmse": 0.23901159112197984, + "mae": 0.13654795289039612, + "r2": 0.6184168457984924, + "score": -0.23901159112197984 + }, + "test": { + "rmse": 0.24670214607411306, + "mae": 0.1664496809244156, + "r2": 0.5841033458709717, + "score": -0.24670214607411306 + } + }, + "time": "0:02:18.814512", + "diversity": { + "train": { + "mean_centered_corr": -0.017886240035295486, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046517129987478256 + }, + "val": { + "mean_centered_corr": -0.019535018131136894, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06728510558605194 + }, + "test": { + "mean_centered_corr": -0.018488358706235886, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08319593966007233 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 21 + ], + "head_selection_time": "0:00:00.015378" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p01-evaluation/0.toml new file mode 100644 index 000000000..85883a50e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p01-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..e87175c62 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p01-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.21727375472083588, + "mae": 0.1175784170627594, + "r2": 0.6984020471572876, + "score": -0.21727375472083588 + }, + "val": { + "rmse": 0.2349593190227014, + "mae": 0.12777242064476013, + "r2": 0.6312460899353027, + "score": -0.2349593190227014 + }, + "test": { + "rmse": 0.23427020750192581, + "mae": 0.14806360006332397, + "r2": 0.6249633431434631, + "score": -0.23427020750192581 + } + }, + "time": "0:02:18.814512", + "diversity": { + "train": { + "mean_centered_corr": -0.017886240035295486, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046517129987478256 + }, + "val": { + "mean_centered_corr": -0.019535018131136894, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06728510558605194 + }, + "test": { + "mean_centered_corr": -0.018488358706235886, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08319593966007233 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..6c0f527a4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,701 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.218914778840735, + "mae": 0.11776337772607803, + "r2": 0.6938289999961853, + "score": -0.218914778840735 + }, + "val": { + "rmse": 0.2334202712313093, + "mae": 0.1272968202829361, + "r2": 0.6360611915588379, + "score": -0.2334202712313093 + }, + "test": { + "rmse": 0.2351349272835184, + "mae": 0.1497408002614975, + "r2": 0.6221896409988403, + "score": -0.2351349272835184 + } + }, + "time": "0:02:18.814512", + "diversity": { + "train": { + "mean_centered_corr": -0.017886240035295486, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046517129987478256 + }, + "val": { + "mean_centered_corr": -0.019535018131136894, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06728510558605194 + }, + "test": { + "mean_centered_corr": -0.018488358706235886, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08319593966007233 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 21, + 7, + 10, + 25, + 19, + 10, + 22, + 6 + ], + "head_selection_time": "0:00:00.131078" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..b20a2e0dc --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24215561133839658, + "mae": 0.1261674165725708, + "r2": 0.625369668006897, + "score": -0.24215561133839658 + }, + "val": { + "rmse": 0.23901917370395387, + "mae": 0.1357598900794983, + "r2": 0.6183926463127136, + "score": -0.23901917370395387 + }, + "test": { + "rmse": 0.23709576833229662, + "mae": 0.1498398780822754, + "r2": 0.6158620715141296, + "score": -0.23709576833229662 + } + }, + "time": "0:02:05.583255", + "diversity": { + "train": { + "mean_centered_corr": -0.01902354508638382, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04608777537941933 + }, + "val": { + "mean_centered_corr": -0.01785912550985813, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06531322002410889 + }, + "test": { + "mean_centered_corr": -0.020982077345252037, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0832035094499588 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015284" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p001-evaluation/0.toml new file mode 100644 index 000000000..f061a28d3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p001-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..a9bf21b1f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p001-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23131163072255229, + "mae": 0.12073468416929245, + "r2": 0.6581710577011108, + "score": -0.23131163072255229 + }, + "val": { + "rmse": 0.23544191074619497, + "mae": 0.1270890235900879, + "r2": 0.6297297477722168, + "score": -0.23544191074619497 + }, + "test": { + "rmse": 0.23248160420213973, + "mae": 0.14418110251426697, + "r2": 0.6306681632995605, + "score": -0.23248160420213973 + } + }, + "time": "0:02:05.583255", + "diversity": { + "train": { + "mean_centered_corr": -0.01902354508638382, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04608777537941933 + }, + "val": { + "mean_centered_corr": -0.01785912550985813, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06531322002410889 + }, + "test": { + "mean_centered_corr": -0.020982077345252037, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0832035094499588 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..84b3884bc --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,697 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23320069974727647, + "mae": 0.1219385489821434, + "r2": 0.6525650024414062, + "score": -0.23320069974727647 + }, + "val": { + "rmse": 0.23354370990991408, + "mae": 0.12699057161808014, + "r2": 0.6356761455535889, + "score": -0.23354370990991408 + }, + "test": { + "rmse": 0.23867420719836574, + "mae": 0.15507498383522034, + "r2": 0.6107302904129028, + "score": -0.23867420719836574 + } + }, + "time": "0:02:05.583255", + "diversity": { + "train": { + "mean_centered_corr": -0.01902354508638382, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04608777537941933 + }, + "val": { + "mean_centered_corr": -0.01785912550985813, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06531322002410889 + }, + "test": { + "mean_centered_corr": -0.020982077345252037, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0832035094499588 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 4, + 0, + 25, + 12 + ], + "head_selection_time": "0:00:00.073888" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..18484cab2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24348651999656226, + "mae": 0.12649968266487122, + "r2": 0.6212403178215027, + "score": -0.24348651999656226 + }, + "val": { + "rmse": 0.238198961838987, + "mae": 0.13287700712680817, + "r2": 0.6210072040557861, + "score": -0.238198961838987 + }, + "test": { + "rmse": 0.23998827851003937, + "mae": 0.15366043150424957, + "r2": 0.6064320802688599, + "score": -0.23998827851003937 + } + }, + "time": "0:02:01.502226", + "diversity": { + "train": { + "mean_centered_corr": -0.01856936514377594, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046877678483724594 + }, + "val": { + "mean_centered_corr": -0.01878707855939865, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06156620755791664 + }, + "test": { + "mean_centered_corr": -0.02026924304664135, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08082668483257294 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015291" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p005-evaluation/0.toml new file mode 100644 index 000000000..824374b5d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p005-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..8ee5f612f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p005-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.230838543618499, + "mae": 0.12101432681083679, + "r2": 0.6595678329467773, + "score": -0.230838543618499 + }, + "val": { + "rmse": 0.23458071263974148, + "mae": 0.1285729557275772, + "r2": 0.6324335336685181, + "score": -0.23458071263974148 + }, + "test": { + "rmse": 0.23412343670770672, + "mae": 0.14895197749137878, + "r2": 0.625433087348938, + "score": -0.23412343670770672 + } + }, + "time": "0:02:01.502226", + "diversity": { + "train": { + "mean_centered_corr": -0.01856936514377594, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046877678483724594 + }, + "val": { + "mean_centered_corr": -0.01878707855939865, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06156620755791664 + }, + "test": { + "mean_centered_corr": -0.02026924304664135, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08082668483257294 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..7ab593e1f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,704 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23149574309806026, + "mae": 0.121489517390728, + "r2": 0.6576266884803772, + "score": -0.23149574309806026 + }, + "val": { + "rmse": 0.2329018389418362, + "mae": 0.12706120312213898, + "r2": 0.6376760005950928, + "score": -0.2329018389418362 + }, + "test": { + "rmse": 0.23679443729395636, + "mae": 0.1538662314414978, + "r2": 0.6168378591537476, + "score": -0.23679443729395636 + } + }, + "time": "0:02:01.502226", + "diversity": { + "train": { + "mean_centered_corr": -0.01856936514377594, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046877678483724594 + }, + "val": { + "mean_centered_corr": -0.01878707855939865, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06156620755791664 + }, + "test": { + "mean_centered_corr": -0.02026924304664135, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08082668483257294 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 4, + 10, + 25, + 12, + 21, + 3, + 25, + 20, + 19, + 12, + 24 + ], + "head_selection_time": "0:00:00.175097" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..8fd234997 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2295723551876263, + "mae": 0.1215803325176239, + "r2": 0.6632922887802124, + "score": -0.2295723551876263 + }, + "val": { + "rmse": 0.2384660660716776, + "mae": 0.14107254147529602, + "r2": 0.6201567649841309, + "score": -0.2384660660716776 + }, + "test": { + "rmse": 0.2608031155268875, + "mae": 0.18369406461715698, + "r2": 0.5352010726928711, + "score": -0.2608031155268875 + } + }, + "time": "0:02:19.607379", + "diversity": { + "train": { + "mean_centered_corr": -0.018292076885700226, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0486440509557724 + }, + "val": { + "mean_centered_corr": -0.01807783916592598, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07439792156219482 + }, + "test": { + "mean_centered_corr": -0.018988410010933876, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09896504878997803 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 25 + ], + "head_selection_time": "0:00:00.015213" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p01-evaluation/0.toml new file mode 100644 index 000000000..253e6a888 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p01-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..23077ce78 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p01-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2176600961009904, + "mae": 0.11412091553211212, + "r2": 0.697328507900238, + "score": -0.2176600961009904 + }, + "val": { + "rmse": 0.23471617503170125, + "mae": 0.12663888931274414, + "r2": 0.6320089101791382, + "score": -0.23471617503170125 + }, + "test": { + "rmse": 0.23405948700639934, + "mae": 0.14594270288944244, + "r2": 0.6256377100944519, + "score": -0.23405948700639934 + } + }, + "time": "0:02:19.607379", + "diversity": { + "train": { + "mean_centered_corr": -0.018292076885700226, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0486440509557724 + }, + "val": { + "mean_centered_corr": -0.01807783916592598, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07439792156219482 + }, + "test": { + "mean_centered_corr": -0.018988410010933876, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09896504878997803 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..531579d5d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,702 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2194825831190966, + "mae": 0.114332415163517, + "r2": 0.6922386884689331, + "score": -0.2194825831190966 + }, + "val": { + "rmse": 0.23344479982230373, + "mae": 0.12650029361248016, + "r2": 0.6359846591949463, + "score": -0.23344479982230373 + }, + "test": { + "rmse": 0.2326324214479151, + "mae": 0.14232482016086578, + "r2": 0.6301887631416321, + "score": -0.2326324214479151 + } + }, + "time": "0:02:19.607379", + "diversity": { + "train": { + "mean_centered_corr": -0.018292076885700226, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0486440509557724 + }, + "val": { + "mean_centered_corr": -0.01807783916592598, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07439792156219482 + }, + "test": { + "mean_centered_corr": -0.018988410010933876, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09896504878997803 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 25, + 7, + 22, + 10, + 26, + 16, + 29, + 17, + 19 + ], + "head_selection_time": "0:00:00.146951" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..ad435cbed --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24039030054595545, + "mae": 0.1237230896949768, + "r2": 0.6308118104934692, + "score": -0.24039030054595545 + }, + "val": { + "rmse": 0.23560357747729638, + "mae": 0.13029368221759796, + "r2": 0.6292210817337036, + "score": -0.23560357747729638 + }, + "test": { + "rmse": 0.2389191702722735, + "mae": 0.15265432000160217, + "r2": 0.6099308729171753, + "score": -0.2389191702722735 + } + }, + "time": "0:01:11.607187", + "diversity": { + "train": { + "mean_centered_corr": -0.018871087580919266, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04625045508146286 + }, + "val": { + "mean_centered_corr": -0.019568709656596184, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0614706352353096 + }, + "test": { + "mean_centered_corr": -0.018940167501568794, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08073103427886963 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015377" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-best-head-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-best-head-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-best-head-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-best-head-evaluation/1/report.json new file mode 100644 index 000000000..df7faffc7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-best-head-evaluation/1/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + }, + "mask_stats": { + "keep_rate_actual": 0.9749681353569031, + "min_features_per_member": 376.0, + "mean_features_per_member": 382.1875, + "max_features_per_member": 388.0, + "min_members_per_feature": 28.0, + "mean_members_per_feature": 31.1989803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9511310051056044, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1184, + "metrics": { + "train": { + "rmse": 0.2171563784853777, + "mae": 0.11889506876468658, + "r2": 0.6987277865409851, + "score": -0.2171563784853777 + }, + "val": { + "rmse": 0.24254141605050564, + "mae": 0.13972081243991852, + "r2": 0.607062816619873, + "score": -0.24254141605050564 + }, + "test": { + "rmse": 0.24204913155172003, + "mae": 0.15291987359523773, + "r2": 0.5996437072753906, + "score": -0.24204913155172003 + } + }, + "time": "0:01:33.897406", + "diversity": { + "train": { + "mean_centered_corr": -0.017405925318598747, + "mean_pairwise_disagreement": NaN, + "member_std": 0.049276966601610184 + }, + "val": { + "mean_centered_corr": -0.017822835594415665, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07913164794445038 + }, + "test": { + "mean_centered_corr": -0.01794285699725151, + "mean_pairwise_disagreement": NaN, + "member_std": 0.10268954932689667 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 27 + ], + "head_selection_time": "0:00:00.015220" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-best-head-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-best-head-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-best-head-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-best-head-evaluation/2/report.json new file mode 100644 index 000000000..996afe498 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-best-head-evaluation/2/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + }, + "mask_stats": { + "keep_rate_actual": 0.9760841727256775, + "min_features_per_member": 372.0, + "mean_features_per_member": 382.625, + "max_features_per_member": 390.0, + "min_members_per_feature": 28.0, + "mean_members_per_feature": 31.23469352722168, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9532826281104112, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2229962847735884, + "mae": 0.12252119928598404, + "r2": 0.6823059320449829, + "score": -0.2229962847735884 + }, + "val": { + "rmse": 0.23847619665119374, + "mae": 0.134144127368927, + "r2": 0.6201244592666626, + "score": -0.23847619665119374 + }, + "test": { + "rmse": 0.2542706881867437, + "mae": 0.17485685646533966, + "r2": 0.5581934452056885, + "score": -0.2542706881867437 + } + }, + "time": "0:01:20.981125", + "diversity": { + "train": { + "mean_centered_corr": -0.018493393436074257, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04774053022265434 + }, + "val": { + "mean_centered_corr": -0.018883541226387024, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07628075778484344 + }, + "test": { + "mean_centered_corr": -0.01829403080046177, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09460576623678207 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 24 + ], + "head_selection_time": "0:00:00.015828" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/0.toml new file mode 100644 index 000000000..3417f3906 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..578c23e0a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/0/report.json @@ -0,0 +1,684 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23278188472143527, + "mae": 0.12037675827741623, + "r2": 0.6538118124008179, + "score": -0.23278188472143527 + }, + "val": { + "rmse": 0.235050071622046, + "mae": 0.12535566091537476, + "r2": 0.6309611797332764, + "score": -0.235050071622046 + }, + "test": { + "rmse": 0.23184680318718037, + "mae": 0.14313000440597534, + "r2": 0.6326823234558105, + "score": -0.23184680318718037 + } + }, + "time": "0:01:11.607187", + "diversity": { + "train": { + "mean_centered_corr": -0.018871087580919266, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04625045508146286 + }, + "val": { + "mean_centered_corr": -0.019568709656596184, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0614706352353096 + }, + "test": { + "mean_centered_corr": -0.018940167501568794, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08073103427886963 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/1.toml new file mode 100644 index 000000000..71d78a574 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/1.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 1 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 1 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/1/report.json new file mode 100644 index 000000000..9c63cb100 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/1/report.json @@ -0,0 +1,684 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + }, + "mask_stats": { + "keep_rate_actual": 0.9749681353569031, + "min_features_per_member": 376.0, + "mean_features_per_member": 382.1875, + "max_features_per_member": 388.0, + "min_members_per_feature": 28.0, + "mean_members_per_feature": 31.1989803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9511310051056044, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1184, + "metrics": { + "train": { + "rmse": 0.20665772048240127, + "mae": 0.11000452935695648, + "r2": 0.7271542549133301, + "score": -0.20665772048240127 + }, + "val": { + "rmse": 0.23618618194018487, + "mae": 0.12565438449382782, + "r2": 0.6273850798606873, + "score": -0.23618618194018487 + }, + "test": { + "rmse": 0.23253236276211406, + "mae": 0.1399872750043869, + "r2": 0.6305068731307983, + "score": -0.23253236276211406 + } + }, + "time": "0:01:33.897406", + "diversity": { + "train": { + "mean_centered_corr": -0.017405925318598747, + "mean_pairwise_disagreement": NaN, + "member_std": 0.049276966601610184 + }, + "val": { + "mean_centered_corr": -0.017822835594415665, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07913164794445038 + }, + "test": { + "mean_centered_corr": -0.01794285699725151, + "mean_pairwise_disagreement": NaN, + "member_std": 0.10268954932689667 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/2.toml new file mode 100644 index 000000000..1cfba881b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/2.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 2 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 2 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/2/report.json new file mode 100644 index 000000000..253e9fb6a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/2/report.json @@ -0,0 +1,684 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + }, + "mask_stats": { + "keep_rate_actual": 0.9760841727256775, + "min_features_per_member": 372.0, + "mean_features_per_member": 382.625, + "max_features_per_member": 390.0, + "min_members_per_feature": 28.0, + "mean_members_per_feature": 31.23469352722168, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9532826281104112, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.21767157152228817, + "mae": 0.11745961010456085, + "r2": 0.6972966194152832, + "score": -0.21767157152228817 + }, + "val": { + "rmse": 0.23592642069691294, + "mae": 0.12567678093910217, + "r2": 0.6282042264938354, + "score": -0.23592642069691294 + }, + "test": { + "rmse": 0.2326191697888986, + "mae": 0.14303357899188995, + "r2": 0.6302309036254883, + "score": -0.2326191697888986 + } + }, + "time": "0:01:20.981125", + "diversity": { + "train": { + "mean_centered_corr": -0.018493393436074257, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04774053022265434 + }, + "val": { + "mean_centered_corr": -0.018883541226387024, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07628075778484344 + }, + "test": { + "mean_centered_corr": -0.01829403080046177, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09460576623678207 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..31384f64a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,692 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23573796221874224, + "mae": 0.12117554992437363, + "r2": 0.6449635028839111, + "score": -0.23573796221874224 + }, + "val": { + "rmse": 0.2324412121462238, + "mae": 0.12522703409194946, + "r2": 0.6391078233718872, + "score": -0.2324412121462238 + }, + "test": { + "rmse": 0.23820740696857753, + "mae": 0.1556985229253769, + "r2": 0.6122515201568604, + "score": -0.23820740696857753 + } + }, + "time": "0:01:11.607187", + "diversity": { + "train": { + "mean_centered_corr": -0.018871087580919266, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04625045508146286 + }, + "val": { + "mean_centered_corr": -0.019568709656596184, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0614706352353096 + }, + "test": { + "mean_centered_corr": -0.018940167501568794, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08073103427886963 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 0, + 1, + 25, + 21 + ], + "head_selection_time": "0:00:00.087943" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-greedy-heads-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-greedy-heads-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-greedy-heads-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-greedy-heads-evaluation/1/report.json new file mode 100644 index 000000000..645dd9e30 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-greedy-heads-evaluation/1/report.json @@ -0,0 +1,703 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + }, + "mask_stats": { + "keep_rate_actual": 0.9749681353569031, + "min_features_per_member": 376.0, + "mean_features_per_member": 382.1875, + "max_features_per_member": 388.0, + "min_members_per_feature": 28.0, + "mean_members_per_feature": 31.1989803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9511310051056044, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1184, + "metrics": { + "train": { + "rmse": 0.20580531574085753, + "mae": 0.11071916669607162, + "r2": 0.7294004559516907, + "score": -0.20580531574085753 + }, + "val": { + "rmse": 0.23397077829789442, + "mae": 0.12587550282478333, + "r2": 0.6343424916267395, + "score": -0.23397077829789442 + }, + "test": { + "rmse": 0.23276530465893142, + "mae": 0.13940665125846863, + "r2": 0.6297661662101746, + "score": -0.23276530465893142 + } + }, + "time": "0:01:33.897406", + "diversity": { + "train": { + "mean_centered_corr": -0.017405925318598747, + "mean_pairwise_disagreement": NaN, + "member_std": 0.049276966601610184 + }, + "val": { + "mean_centered_corr": -0.017822835594415665, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07913164794445038 + }, + "test": { + "mean_centered_corr": -0.01794285699725151, + "mean_pairwise_disagreement": NaN, + "member_std": 0.10268954932689667 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 27, + 6, + 1, + 15, + 2, + 13, + 0, + 6, + 1, + 29, + 31, + 5, + 1, + 21, + 6, + 22 + ], + "head_selection_time": "0:00:00.252986" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-greedy-heads-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-greedy-heads-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-greedy-heads-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-greedy-heads-evaluation/2/report.json new file mode 100644 index 000000000..1ecac8196 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-greedy-heads-evaluation/2/report.json @@ -0,0 +1,696 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + }, + "mask_stats": { + "keep_rate_actual": 0.9760841727256775, + "min_features_per_member": 372.0, + "mean_features_per_member": 382.625, + "max_features_per_member": 390.0, + "min_members_per_feature": 28.0, + "mean_members_per_feature": 31.23469352722168, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9532826281104112, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2210309033660134, + "mae": 0.12036030739545822, + "r2": 0.6878812313079834, + "score": -0.2210309033660134 + }, + "val": { + "rmse": 0.23411027740292373, + "mae": 0.128285214304924, + "r2": 0.633906364440918, + "score": -0.23411027740292373 + }, + "test": { + "rmse": 0.24496736545125247, + "mae": 0.16705775260925293, + "r2": 0.589931845664978, + "score": -0.24496736545125247 + } + }, + "time": "0:01:20.981125", + "diversity": { + "train": { + "mean_centered_corr": -0.018493393436074257, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04774053022265434 + }, + "val": { + "mean_centered_corr": -0.018883541226387024, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07628075778484344 + }, + "test": { + "mean_centered_corr": -0.01829403080046177, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09460576623678207 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 24, + 4, + 27, + 6, + 3, + 30, + 5, + 27, + 4 + ], + "head_selection_time": "0:00:00.147559" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..022c71440 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24039030054595545, + "mae": 0.1237230896949768, + "r2": 0.6308118104934692, + "score": -0.24039030054595545 + }, + "val": { + "rmse": 0.23560357747729638, + "mae": 0.13029368221759796, + "r2": 0.6292210817337036, + "score": -0.23560357747729638 + }, + "test": { + "rmse": 0.2389191702722735, + "mae": 0.15265432000160217, + "r2": 0.6099308729171753, + "score": -0.2389191702722735 + } + }, + "time": "0:01:13.814362", + "diversity": { + "train": { + "mean_centered_corr": -0.018871087580919266, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04625045508146286 + }, + "val": { + "mean_centered_corr": -0.019568709656596184, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0614706352353096 + }, + "test": { + "mean_centered_corr": -0.018940167501568794, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08073103427886963 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015454" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-best-head-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-best-head-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-best-head-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-best-head-evaluation/1/report.json new file mode 100644 index 000000000..051fe52d7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-best-head-evaluation/1/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + }, + "mask_stats": { + "keep_rate_actual": 0.9749681353569031, + "min_features_per_member": 376.0, + "mean_features_per_member": 382.1875, + "max_features_per_member": 388.0, + "min_members_per_feature": 28.0, + "mean_members_per_feature": 31.1989803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9511310051056044, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1184, + "metrics": { + "train": { + "rmse": 0.2171563784853777, + "mae": 0.11889506876468658, + "r2": 0.6987277865409851, + "score": -0.2171563784853777 + }, + "val": { + "rmse": 0.24254141605050564, + "mae": 0.13972081243991852, + "r2": 0.607062816619873, + "score": -0.24254141605050564 + }, + "test": { + "rmse": 0.24204913155172003, + "mae": 0.15291987359523773, + "r2": 0.5996437072753906, + "score": -0.24204913155172003 + } + }, + "time": "0:01:33.744311", + "diversity": { + "train": { + "mean_centered_corr": -0.017405925318598747, + "mean_pairwise_disagreement": NaN, + "member_std": 0.049276966601610184 + }, + "val": { + "mean_centered_corr": -0.017822835594415665, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07913164794445038 + }, + "test": { + "mean_centered_corr": -0.01794285699725151, + "mean_pairwise_disagreement": NaN, + "member_std": 0.10268954932689667 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 27 + ], + "head_selection_time": "0:00:00.015010" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-best-head-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-best-head-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-best-head-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-best-head-evaluation/2/report.json new file mode 100644 index 000000000..bc47fc492 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-best-head-evaluation/2/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + }, + "mask_stats": { + "keep_rate_actual": 0.9760841727256775, + "min_features_per_member": 372.0, + "mean_features_per_member": 382.625, + "max_features_per_member": 390.0, + "min_members_per_feature": 28.0, + "mean_members_per_feature": 31.23469352722168, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9532826281104112, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2229962847735884, + "mae": 0.12252119928598404, + "r2": 0.6823059320449829, + "score": -0.2229962847735884 + }, + "val": { + "rmse": 0.23847619665119374, + "mae": 0.134144127368927, + "r2": 0.6201244592666626, + "score": -0.23847619665119374 + }, + "test": { + "rmse": 0.2542706881867437, + "mae": 0.17485685646533966, + "r2": 0.5581934452056885, + "score": -0.2542706881867437 + } + }, + "time": "0:01:19.517367", + "diversity": { + "train": { + "mean_centered_corr": -0.018493393436074257, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04774053022265434 + }, + "val": { + "mean_centered_corr": -0.018883541226387024, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07628075778484344 + }, + "test": { + "mean_centered_corr": -0.01829403080046177, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09460576623678207 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 24 + ], + "head_selection_time": "0:00:00.015325" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/0.toml new file mode 100644 index 000000000..aff429e28 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..193998f2e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/0/report.json @@ -0,0 +1,684 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23278188472143527, + "mae": 0.12037675827741623, + "r2": 0.6538118124008179, + "score": -0.23278188472143527 + }, + "val": { + "rmse": 0.235050071622046, + "mae": 0.12535566091537476, + "r2": 0.6309611797332764, + "score": -0.235050071622046 + }, + "test": { + "rmse": 0.23184680318718037, + "mae": 0.14313000440597534, + "r2": 0.6326823234558105, + "score": -0.23184680318718037 + } + }, + "time": "0:01:13.814362", + "diversity": { + "train": { + "mean_centered_corr": -0.018871087580919266, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04625045508146286 + }, + "val": { + "mean_centered_corr": -0.019568709656596184, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0614706352353096 + }, + "test": { + "mean_centered_corr": -0.018940167501568794, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08073103427886963 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/1.toml new file mode 100644 index 000000000..ad01c71b3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/1.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 1 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 1 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/1/report.json new file mode 100644 index 000000000..f4c5d18c3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/1/report.json @@ -0,0 +1,684 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + }, + "mask_stats": { + "keep_rate_actual": 0.9749681353569031, + "min_features_per_member": 376.0, + "mean_features_per_member": 382.1875, + "max_features_per_member": 388.0, + "min_members_per_feature": 28.0, + "mean_members_per_feature": 31.1989803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9511310051056044, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1184, + "metrics": { + "train": { + "rmse": 0.20665772048240127, + "mae": 0.11000452935695648, + "r2": 0.7271542549133301, + "score": -0.20665772048240127 + }, + "val": { + "rmse": 0.23618618194018487, + "mae": 0.12565438449382782, + "r2": 0.6273850798606873, + "score": -0.23618618194018487 + }, + "test": { + "rmse": 0.23253236276211406, + "mae": 0.1399872750043869, + "r2": 0.6305068731307983, + "score": -0.23253236276211406 + } + }, + "time": "0:01:33.744311", + "diversity": { + "train": { + "mean_centered_corr": -0.017405925318598747, + "mean_pairwise_disagreement": NaN, + "member_std": 0.049276966601610184 + }, + "val": { + "mean_centered_corr": -0.017822835594415665, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07913164794445038 + }, + "test": { + "mean_centered_corr": -0.01794285699725151, + "mean_pairwise_disagreement": NaN, + "member_std": 0.10268954932689667 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/2.toml new file mode 100644 index 000000000..86addbb91 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/2.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 2 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 2 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/2/report.json new file mode 100644 index 000000000..65d6d6497 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/2/report.json @@ -0,0 +1,684 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + }, + "mask_stats": { + "keep_rate_actual": 0.9760841727256775, + "min_features_per_member": 372.0, + "mean_features_per_member": 382.625, + "max_features_per_member": 390.0, + "min_members_per_feature": 28.0, + "mean_members_per_feature": 31.23469352722168, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9532826281104112, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.21767157152228817, + "mae": 0.11745961010456085, + "r2": 0.6972966194152832, + "score": -0.21767157152228817 + }, + "val": { + "rmse": 0.23592642069691294, + "mae": 0.12567678093910217, + "r2": 0.6282042264938354, + "score": -0.23592642069691294 + }, + "test": { + "rmse": 0.2326191697888986, + "mae": 0.14303357899188995, + "r2": 0.6302309036254883, + "score": -0.2326191697888986 + } + }, + "time": "0:01:19.517367", + "diversity": { + "train": { + "mean_centered_corr": -0.018493393436074257, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04774053022265434 + }, + "val": { + "mean_centered_corr": -0.018883541226387024, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07628075778484344 + }, + "test": { + "mean_centered_corr": -0.01829403080046177, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09460576623678207 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..55f69c783 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,692 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23573796221874224, + "mae": 0.12117554992437363, + "r2": 0.6449635028839111, + "score": -0.23573796221874224 + }, + "val": { + "rmse": 0.2324412121462238, + "mae": 0.12522703409194946, + "r2": 0.6391078233718872, + "score": -0.2324412121462238 + }, + "test": { + "rmse": 0.23820740696857753, + "mae": 0.1556985229253769, + "r2": 0.6122515201568604, + "score": -0.23820740696857753 + } + }, + "time": "0:01:13.814362", + "diversity": { + "train": { + "mean_centered_corr": -0.018871087580919266, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04625045508146286 + }, + "val": { + "mean_centered_corr": -0.019568709656596184, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0614706352353096 + }, + "test": { + "mean_centered_corr": -0.018940167501568794, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08073103427886963 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 0, + 1, + 25, + 21 + ], + "head_selection_time": "0:00:00.088240" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-greedy-heads-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-greedy-heads-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-greedy-heads-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-greedy-heads-evaluation/1/report.json new file mode 100644 index 000000000..9dcd68238 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-greedy-heads-evaluation/1/report.json @@ -0,0 +1,703 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + }, + "mask_stats": { + "keep_rate_actual": 0.9749681353569031, + "min_features_per_member": 376.0, + "mean_features_per_member": 382.1875, + "max_features_per_member": 388.0, + "min_members_per_feature": 28.0, + "mean_members_per_feature": 31.1989803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9511310051056044, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1184, + "metrics": { + "train": { + "rmse": 0.20580531574085753, + "mae": 0.11071916669607162, + "r2": 0.7294004559516907, + "score": -0.20580531574085753 + }, + "val": { + "rmse": 0.23397077829789442, + "mae": 0.12587550282478333, + "r2": 0.6343424916267395, + "score": -0.23397077829789442 + }, + "test": { + "rmse": 0.23276530465893142, + "mae": 0.13940665125846863, + "r2": 0.6297661662101746, + "score": -0.23276530465893142 + } + }, + "time": "0:01:33.744311", + "diversity": { + "train": { + "mean_centered_corr": -0.017405925318598747, + "mean_pairwise_disagreement": NaN, + "member_std": 0.049276966601610184 + }, + "val": { + "mean_centered_corr": -0.017822835594415665, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07913164794445038 + }, + "test": { + "mean_centered_corr": -0.01794285699725151, + "mean_pairwise_disagreement": NaN, + "member_std": 0.10268954932689667 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 27, + 6, + 1, + 15, + 2, + 13, + 0, + 6, + 1, + 29, + 31, + 5, + 1, + 21, + 6, + 22 + ], + "head_selection_time": "0:00:00.247440" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-greedy-heads-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-greedy-heads-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-greedy-heads-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-greedy-heads-evaluation/2/report.json new file mode 100644 index 000000000..88ac1a9ce --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-greedy-heads-evaluation/2/report.json @@ -0,0 +1,696 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + }, + "mask_stats": { + "keep_rate_actual": 0.9760841727256775, + "min_features_per_member": 372.0, + "mean_features_per_member": 382.625, + "max_features_per_member": 390.0, + "min_members_per_feature": 28.0, + "mean_members_per_feature": 31.23469352722168, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9532826281104112, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2210309033660134, + "mae": 0.12036030739545822, + "r2": 0.6878812313079834, + "score": -0.2210309033660134 + }, + "val": { + "rmse": 0.23411027740292373, + "mae": 0.128285214304924, + "r2": 0.633906364440918, + "score": -0.23411027740292373 + }, + "test": { + "rmse": 0.24496736545125247, + "mae": 0.16705775260925293, + "r2": 0.589931845664978, + "score": -0.24496736545125247 + } + }, + "time": "0:01:19.517367", + "diversity": { + "train": { + "mean_centered_corr": -0.018493393436074257, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04774053022265434 + }, + "val": { + "mean_centered_corr": -0.018883541226387024, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07628075778484344 + }, + "test": { + "mean_centered_corr": -0.01829403080046177, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09460576623678207 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 24, + 4, + 27, + 6, + 3, + 30, + 5, + 27, + 4 + ], + "head_selection_time": "0:00:00.145674" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/all_four_combined-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/all_four_combined-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/all_four_combined-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/all_four_combined-best-head-evaluation/0/report.json new file mode 100644 index 000000000..6bf7e0a6e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/all_four_combined-best-head-evaluation/0/report.json @@ -0,0 +1,455 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9000651240348816, + "min_features_per_member": 164.0, + "mean_features_per_member": 172.8125, + "max_features_per_member": 181.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.80208396911621, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8173701129083908, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/cooking-time/all_four_combined-evaluation/_esam_diagnostics/all_four_combined-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.4515053148590503, + "mae": 0.35153692960739136, + "r2": 0.5317407250404358, + "score": -0.4515053148590503 + }, + "val": { + "rmse": 0.46391938568359065, + "mae": 0.36074763536453247, + "r2": 0.5064277052879333, + "score": -0.46391938568359065 + }, + "test": { + "rmse": 0.48142889277254225, + "mae": 0.3758453130722046, + "r2": 0.4832448363304138, + "score": -0.48142889277254225 + } + }, + "time": "0:38:28.268413", + "diversity": { + "train": { + "mean_centered_corr": -0.009637685492634773, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04498574882745743 + }, + "val": { + "mean_centered_corr": -0.009298564866185188, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046711746603250504 + }, + "test": { + "mean_centered_corr": -0.00908930879086256, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04780808091163635 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 25 + ], + "head_selection_time": "0:00:00.028059" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/all_four_combined-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/all_four_combined-evaluation/0.toml new file mode 100644 index 000000000..7de7ee9ce --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/all_four_combined-evaluation/0.toml @@ -0,0 +1,74 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 8 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "cooking-time" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/all_four_combined-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/all_four_combined-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/all_four_combined-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/all_four_combined-evaluation/0/report.json new file mode 100644 index 000000000..eca7ffcc8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/all_four_combined-evaluation/0/report.json @@ -0,0 +1,451 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9000651240348816, + "min_features_per_member": 164.0, + "mean_features_per_member": 172.8125, + "max_features_per_member": 181.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.80208396911621, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8173701129083908, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/cooking-time/all_four_combined-evaluation/_esam_diagnostics/all_four_combined-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.44819067366049176, + "mae": 0.3488732874393463, + "r2": 0.5385907888412476, + "score": -0.44819067366049176 + }, + "val": { + "rmse": 0.463355012524859, + "mae": 0.3601790964603424, + "r2": 0.5076278448104858, + "score": -0.463355012524859 + }, + "test": { + "rmse": 0.4809202538385541, + "mae": 0.3753686547279358, + "r2": 0.48433613777160645, + "score": -0.4809202538385541 + } + }, + "time": "0:38:28.268413", + "diversity": { + "train": { + "mean_centered_corr": -0.009637685492634773, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04498574882745743 + }, + "val": { + "mean_centered_corr": -0.009298564866185188, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046711746603250504 + }, + "test": { + "mean_centered_corr": -0.00908930879086256, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04780808091163635 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/all_four_combined-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/all_four_combined-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/all_four_combined-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/all_four_combined-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..f61c80709 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/all_four_combined-greedy-heads-evaluation/0/report.json @@ -0,0 +1,464 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9000651240348816, + "min_features_per_member": 164.0, + "mean_features_per_member": 172.8125, + "max_features_per_member": 181.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.80208396911621, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8173701129083908, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/cooking-time/all_four_combined-evaluation/_esam_diagnostics/all_four_combined-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.44759809638897813, + "mae": 0.3485293388366699, + "r2": 0.539810061454773, + "score": -0.44759809638897813 + }, + "val": { + "rmse": 0.4629352081332738, + "mae": 0.35992681980133057, + "r2": 0.5085196495056152, + "score": -0.4629352081332738 + }, + "test": { + "rmse": 0.4802514336856013, + "mae": 0.3748669922351837, + "r2": 0.48576945066452026, + "score": -0.4802514336856013 + } + }, + "time": "0:38:28.268413", + "diversity": { + "train": { + "mean_centered_corr": -0.009637685492634773, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04498574882745743 + }, + "val": { + "mean_centered_corr": -0.009298564866185188, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046711746603250504 + }, + "test": { + "mean_centered_corr": -0.00908930879086256, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04780808091163635 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 25, + 24, + 19, + 1, + 21, + 27, + 15, + 2, + 12, + 31 + ], + "head_selection_time": "0:00:00.356165" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/baseline_plr-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/baseline_plr-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/baseline_plr-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/baseline_plr-best-head-evaluation/0/report.json new file mode 100644 index 000000000..dca12e41a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/baseline_plr-best-head-evaluation/0/report.json @@ -0,0 +1,147 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44650602179756516, + "mae": 0.34757497906684875, + "r2": 0.5420529246330261, + "score": -0.44650602179756516 + }, + "val": { + "rmse": 0.4652274815891859, + "mae": 0.3617282807826996, + "r2": 0.503640353679657, + "score": -0.4652274815891859 + }, + "test": { + "rmse": 0.4820660657381116, + "mae": 0.3761262893676758, + "r2": 0.48187607526779175, + "score": -0.4820660657381116 + } + }, + "time": "0:09:50.236903", + "diversity": { + "train": { + "mean_centered_corr": -0.020426616072654724, + "mean_pairwise_disagreement": NaN, + "member_std": 0.051189813762903214 + }, + "val": { + "mean_centered_corr": -0.02039431221783161, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05273108929395676 + }, + "test": { + "mean_centered_corr": -0.020482471212744713, + "mean_pairwise_disagreement": NaN, + "member_std": 0.053407639265060425 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 27 + ], + "head_selection_time": "0:00:00.028691" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/baseline_plr-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/baseline_plr-evaluation/0.toml new file mode 100644 index 000000000..78c613dfb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/baseline_plr-evaluation/0.toml @@ -0,0 +1,33 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/baseline_plr-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/baseline_plr-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/baseline_plr-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/baseline_plr-evaluation/0/report.json new file mode 100644 index 000000000..2c476e8cd --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/baseline_plr-evaluation/0/report.json @@ -0,0 +1,143 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44389690465868414, + "mae": 0.34541288018226624, + "r2": 0.5473892688751221, + "score": -0.44389690465868414 + }, + "val": { + "rmse": 0.4627596029926704, + "mae": 0.35959550738334656, + "r2": 0.5088924169540405, + "score": -0.4627596029926704 + }, + "test": { + "rmse": 0.4802386655626689, + "mae": 0.37469902634620667, + "r2": 0.4857967495918274, + "score": -0.4802386655626689 + } + }, + "time": "0:09:50.236903", + "diversity": { + "train": { + "mean_centered_corr": -0.020426616072654724, + "mean_pairwise_disagreement": NaN, + "member_std": 0.051189813762903214 + }, + "val": { + "mean_centered_corr": -0.02039431221783161, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05273108929395676 + }, + "test": { + "mean_centered_corr": -0.020482471212744713, + "mean_pairwise_disagreement": NaN, + "member_std": 0.053407639265060425 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/baseline_plr-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/baseline_plr-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/baseline_plr-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/baseline_plr-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..81f6192e3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/baseline_plr-greedy-heads-evaluation/0/report.json @@ -0,0 +1,157 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44394242184625543, + "mae": 0.3455282151699066, + "r2": 0.547296404838562, + "score": -0.44394242184625543 + }, + "val": { + "rmse": 0.4625169879783321, + "mae": 0.3595449924468994, + "r2": 0.5094072818756104, + "score": -0.4625169879783321 + }, + "test": { + "rmse": 0.47990786370986477, + "mae": 0.3745970129966736, + "r2": 0.4865049123764038, + "score": -0.47990786370986477 + } + }, + "time": "0:09:50.236903", + "diversity": { + "train": { + "mean_centered_corr": -0.020426616072654724, + "mean_pairwise_disagreement": NaN, + "member_std": 0.051189813762903214 + }, + "val": { + "mean_centered_corr": -0.02039431221783161, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05273108929395676 + }, + "test": { + "mean_centered_corr": -0.020482471212744713, + "mean_pairwise_disagreement": NaN, + "member_std": 0.053407639265060425 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 27, + 17, + 5, + 3, + 14, + 4, + 22, + 6, + 24, + 10, + 16 + ], + "head_selection_time": "0:00:00.401983" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/cf_fisd_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/cf_fisd_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/cf_fisd_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/cf_fisd_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..ac492b73b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/cf_fisd_only-best-head-evaluation/0/report.json @@ -0,0 +1,400 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.45293358344227996, + "mae": 0.35263803601264954, + "r2": 0.5287735462188721, + "score": -0.45293358344227996 + }, + "val": { + "rmse": 0.46397138529545745, + "mae": 0.3607122600078583, + "r2": 0.5063170194625854, + "score": -0.46397138529545745 + }, + "test": { + "rmse": 0.4813020969034895, + "mae": 0.3759235143661499, + "r2": 0.48351699113845825, + "score": -0.4813020969034895 + } + }, + "time": "0:10:10.940252", + "diversity": { + "train": { + "mean_centered_corr": -0.004884317051619291, + "mean_pairwise_disagreement": NaN, + "member_std": 0.042778123170137405 + }, + "val": { + "mean_centered_corr": -0.005044749937951565, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04371502622961998 + }, + "test": { + "mean_centered_corr": -0.005197949707508087, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04438793286681175 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 2 + ], + "head_selection_time": "0:00:00.027493" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/cf_fisd_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/cf_fisd_only-evaluation/0.toml new file mode 100644 index 000000000..a22753f01 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/cf_fisd_only-evaluation/0.toml @@ -0,0 +1,44 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "cooking-time" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/cf_fisd_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/cf_fisd_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/cf_fisd_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/cf_fisd_only-evaluation/0/report.json new file mode 100644 index 000000000..6635035ec --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/cf_fisd_only-evaluation/0/report.json @@ -0,0 +1,396 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.4484508087097321, + "mae": 0.34904322028160095, + "r2": 0.5380549430847168, + "score": -0.4484508087097321 + }, + "val": { + "rmse": 0.4632809116304394, + "mae": 0.36012470722198486, + "r2": 0.5077853202819824, + "score": -0.4632809116304394 + }, + "test": { + "rmse": 0.48091704691326154, + "mae": 0.3753390908241272, + "r2": 0.4843430519104004, + "score": -0.48091704691326154 + } + }, + "time": "0:10:10.940252", + "diversity": { + "train": { + "mean_centered_corr": -0.004884317051619291, + "mean_pairwise_disagreement": NaN, + "member_std": 0.042778123170137405 + }, + "val": { + "mean_centered_corr": -0.005044749937951565, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04371502622961998 + }, + "test": { + "mean_centered_corr": -0.005197949707508087, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04438793286681175 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/cf_fisd_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/cf_fisd_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/cf_fisd_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/cf_fisd_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..8e51c0a26 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/cf_fisd_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,403 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.44943641696524117, + "mae": 0.34985706210136414, + "r2": 0.5360222458839417, + "score": -0.44943641696524117 + }, + "val": { + "rmse": 0.4629530240908241, + "mae": 0.35998642444610596, + "r2": 0.5084818601608276, + "score": -0.4629530240908241 + }, + "test": { + "rmse": 0.480628211999089, + "mae": 0.37514716386795044, + "r2": 0.48496222496032715, + "score": -0.480628211999089 + } + }, + "time": "0:10:10.940252", + "diversity": { + "train": { + "mean_centered_corr": -0.004884317051619291, + "mean_pairwise_disagreement": NaN, + "member_std": 0.042778123170137405 + }, + "val": { + "mean_centered_corr": -0.005044749937951565, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04371502622961998 + }, + "test": { + "mean_centered_corr": -0.005197949707508087, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04438793286681175 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 2, + 14, + 24, + 31 + ], + "head_selection_time": "0:00:00.139986" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/esam_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/esam_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/esam_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/esam_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..7a1f0e5ca --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/esam_only-best-head-evaluation/0/report.json @@ -0,0 +1,157 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/cooking-time/esam_only-evaluation/_esam_diagnostics/esam_only-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4481166090239535, + "mae": 0.3489066958427429, + "r2": 0.538743257522583, + "score": -0.4481166090239535 + }, + "val": { + "rmse": 0.4651283707110853, + "mae": 0.3615894913673401, + "r2": 0.5038518309593201, + "score": -0.4651283707110853 + }, + "test": { + "rmse": 0.4823068025931435, + "mae": 0.37674346566200256, + "r2": 0.48135846853256226, + "score": -0.4823068025931435 + } + }, + "time": "0:15:54.265201", + "diversity": { + "train": { + "mean_centered_corr": -0.020383337512612343, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05032477155327797 + }, + "val": { + "mean_centered_corr": -0.02031499706208706, + "mean_pairwise_disagreement": NaN, + "member_std": 0.051892586052417755 + }, + "test": { + "mean_centered_corr": -0.02030996046960354, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05258215218782425 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 3 + ], + "head_selection_time": "0:00:00.027704" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/esam_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/esam_only-evaluation/0.toml new file mode 100644 index 000000000..ae52f3ec3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/esam_only-evaluation/0.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/esam_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/esam_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/esam_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/esam_only-evaluation/0/report.json new file mode 100644 index 000000000..94fa2204e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/esam_only-evaluation/0/report.json @@ -0,0 +1,153 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/cooking-time/esam_only-evaluation/_esam_diagnostics/esam_only-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44408822333443154, + "mae": 0.3455769121646881, + "r2": 0.5469989776611328, + "score": -0.44408822333443154 + }, + "val": { + "rmse": 0.4628077726618535, + "mae": 0.3596377372741699, + "r2": 0.5087901949882507, + "score": -0.4628077726618535 + }, + "test": { + "rmse": 0.48022789849772834, + "mae": 0.37472617626190186, + "r2": 0.48581981658935547, + "score": -0.48022789849772834 + } + }, + "time": "0:15:54.265201", + "diversity": { + "train": { + "mean_centered_corr": -0.020383337512612343, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05032477155327797 + }, + "val": { + "mean_centered_corr": -0.02031499706208706, + "mean_pairwise_disagreement": NaN, + "member_std": 0.051892586052417755 + }, + "test": { + "mean_centered_corr": -0.02030996046960354, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05258215218782425 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/esam_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/esam_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/esam_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/esam_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..398e51916 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/esam_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,167 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/cooking-time/esam_only-evaluation/_esam_diagnostics/esam_only-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4439705320848652, + "mae": 0.34557339549064636, + "r2": 0.5472390651702881, + "score": -0.4439705320848652 + }, + "val": { + "rmse": 0.4625398779765968, + "mae": 0.35953423380851746, + "r2": 0.5093587040901184, + "score": -0.4625398779765968 + }, + "test": { + "rmse": 0.4797024392153251, + "mae": 0.3744461238384247, + "r2": 0.48694443702697754, + "score": -0.4797024392153251 + } + }, + "time": "0:15:54.265201", + "diversity": { + "train": { + "mean_centered_corr": -0.020383337512612343, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05032477155327797 + }, + "val": { + "mean_centered_corr": -0.02031499706208706, + "mean_pairwise_disagreement": NaN, + "member_std": 0.051892586052417755 + }, + "test": { + "mean_centered_corr": -0.02030996046960354, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05258215218782425 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 3, + 17, + 5, + 27, + 22, + 14, + 10, + 4, + 24, + 25, + 20 + ], + "head_selection_time": "0:00:00.400987" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/mfb_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/mfb_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/mfb_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/mfb_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..3f8cf5f28 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/mfb_only-best-head-evaluation/0/report.json @@ -0,0 +1,183 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9000651240348816, + "min_features_per_member": 164.0, + "mean_features_per_member": 172.8125, + "max_features_per_member": 181.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.80208396911621, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8173701129083908, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44680861066629424, + "mae": 0.3479525148868561, + "r2": 0.5414320230484009, + "score": -0.44680861066629424 + }, + "val": { + "rmse": 0.465403548325863, + "mae": 0.3621026575565338, + "r2": 0.5032646059989929, + "score": -0.465403548325863 + }, + "test": { + "rmse": 0.48311284423774803, + "mae": 0.377098023891449, + "r2": 0.4796234369277954, + "score": -0.48311284423774803 + } + }, + "time": "0:12:03.727238", + "diversity": { + "train": { + "mean_centered_corr": -0.020108899101614952, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05420300364494324 + }, + "val": { + "mean_centered_corr": -0.01979835145175457, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05661233514547348 + }, + "test": { + "mean_centered_corr": -0.01982581429183483, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05757860466837883 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 17 + ], + "head_selection_time": "0:00:00.027716" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/mfb_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/mfb_only-evaluation/0.toml new file mode 100644 index 000000000..349df099f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/mfb_only-evaluation/0.toml @@ -0,0 +1,46 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/mfb_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/mfb_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/mfb_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/mfb_only-evaluation/0/report.json new file mode 100644 index 000000000..27ffe0cdb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/mfb_only-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9000651240348816, + "min_features_per_member": 164.0, + "mean_features_per_member": 172.8125, + "max_features_per_member": 181.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.80208396911621, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8173701129083908, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4438062088250124, + "mae": 0.34538015723228455, + "r2": 0.5475741624832153, + "score": -0.4438062088250124 + }, + "val": { + "rmse": 0.46290981075910853, + "mae": 0.3597574234008789, + "r2": 0.508573591709137, + "score": -0.46290981075910853 + }, + "test": { + "rmse": 0.480415480847424, + "mae": 0.37490591406822205, + "r2": 0.48541808128356934, + "score": -0.480415480847424 + } + }, + "time": "0:12:03.727238", + "diversity": { + "train": { + "mean_centered_corr": -0.020108899101614952, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05420300364494324 + }, + "val": { + "mean_centered_corr": -0.01979835145175457, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05661233514547348 + }, + "test": { + "mean_centered_corr": -0.01982581429183483, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05757860466837883 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/mfb_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/mfb_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/mfb_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/mfb_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..be582a713 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/mfb_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,193 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9000651240348816, + "min_features_per_member": 164.0, + "mean_features_per_member": 172.8125, + "max_features_per_member": 181.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.80208396911621, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8173701129083908, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4439127657129147, + "mae": 0.3454906642436981, + "r2": 0.5473568439483643, + "score": -0.4439127657129147 + }, + "val": { + "rmse": 0.4625593198933149, + "mae": 0.35958266258239746, + "r2": 0.5093174576759338, + "score": -0.4625593198933149 + }, + "test": { + "rmse": 0.479899744054174, + "mae": 0.37457266449928284, + "r2": 0.4865223169326782, + "score": -0.479899744054174 + } + }, + "time": "0:12:03.727238", + "diversity": { + "train": { + "mean_centered_corr": -0.020108899101614952, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05420300364494324 + }, + "val": { + "mean_centered_corr": -0.01979835145175457, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05661233514547348 + }, + "test": { + "mean_centered_corr": -0.01982581429183483, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05757860466837883 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 17, + 14, + 11, + 24, + 25, + 5, + 4, + 27, + 20, + 1, + 9 + ], + "head_selection_time": "0:00:00.396117" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/rla_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/rla_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/rla_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/rla_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..30254cc4d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/rla_only-best-head-evaluation/0/report.json @@ -0,0 +1,156 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44760284038640463, + "mae": 0.3484247922897339, + "r2": 0.5398002862930298, + "score": -0.44760284038640463 + }, + "val": { + "rmse": 0.4651183431272534, + "mae": 0.3614919185638428, + "r2": 0.5038732290267944, + "score": -0.4651183431272534 + }, + "test": { + "rmse": 0.48213882482447107, + "mae": 0.37617027759552, + "r2": 0.48171961307525635, + "score": -0.48213882482447107 + } + }, + "time": "0:23:04.537939", + "diversity": { + "train": { + "mean_centered_corr": -0.020404240116477013, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0513404980301857 + }, + "val": { + "mean_centered_corr": -0.02041091024875641, + "mean_pairwise_disagreement": NaN, + "member_std": 0.052927520126104355 + }, + "test": { + "mean_centered_corr": -0.020323792472481728, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05353504791855812 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 7 + ], + "head_selection_time": "0:00:00.027790" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/rla_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/rla_only-evaluation/0.toml new file mode 100644 index 000000000..17b62a97a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/rla_only-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/rla_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/rla_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/rla_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/rla_only-evaluation/0/report.json new file mode 100644 index 000000000..ae0abfdfd --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/rla_only-evaluation/0/report.json @@ -0,0 +1,152 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4435648840971451, + "mae": 0.34517785906791687, + "r2": 0.5480660200119019, + "score": -0.4435648840971451 + }, + "val": { + "rmse": 0.4627297681263241, + "mae": 0.35958367586135864, + "r2": 0.5089557766914368, + "score": -0.4627297681263241 + }, + "test": { + "rmse": 0.48025858554781287, + "mae": 0.3747011125087738, + "r2": 0.485754132270813, + "score": -0.48025858554781287 + } + }, + "time": "0:23:04.537939", + "diversity": { + "train": { + "mean_centered_corr": -0.020404240116477013, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0513404980301857 + }, + "val": { + "mean_centered_corr": -0.02041091024875641, + "mean_pairwise_disagreement": NaN, + "member_std": 0.052927520126104355 + }, + "test": { + "mean_centered_corr": -0.020323792472481728, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05353504791855812 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/rla_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/rla_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/rla_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/rla_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..551d9ab7f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/rla_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,177 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44353548828929634, + "mae": 0.3452431261539459, + "r2": 0.5481259822845459, + "score": -0.44353548828929634 + }, + "val": { + "rmse": 0.4624375326912171, + "mae": 0.3594800531864166, + "r2": 0.5095758438110352, + "score": -0.4624375326912171 + }, + "test": { + "rmse": 0.48004674599435365, + "mae": 0.3747219443321228, + "r2": 0.48620766401290894, + "score": -0.48004674599435365 + } + }, + "time": "0:23:04.537939", + "diversity": { + "train": { + "mean_centered_corr": -0.020404240116477013, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0513404980301857 + }, + "val": { + "mean_centered_corr": -0.02041091024875641, + "mean_pairwise_disagreement": NaN, + "member_std": 0.052927520126104355 + }, + "test": { + "mean_centered_corr": -0.020323792472481728, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05353504791855812 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 7, + 24, + 17, + 14, + 5, + 3, + 16, + 4, + 10, + 6, + 22, + 17, + 27, + 24, + 21, + 5, + 14, + 31, + 17, + 25, + 24, + 28 + ], + "head_selection_time": "0:00:01.021222" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/all_four_combined-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/all_four_combined-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/all_four_combined-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/all_four_combined-best-head-evaluation/0/report.json new file mode 100644 index 000000000..487266694 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/all_four_combined-best-head-evaluation/0/report.json @@ -0,0 +1,506 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9009943008422852, + "min_features_per_member": 188.0, + "mean_features_per_member": 198.21875, + "max_features_per_member": 206.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.831817626953125, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8196317439934785, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/all_four_combined-evaluation/_esam_diagnostics/all_four_combined-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5163498609634084, + "mae": 0.40700992941856384, + "r2": 0.500271201133728, + "score": -0.5163498609634084 + }, + "val": { + "rmse": 0.557107642501001, + "mae": 0.44366100430488586, + "r2": 0.4147301912307739, + "score": -0.557107642501001 + }, + "test": { + "rmse": 0.5520572896128711, + "mae": 0.4460996389389038, + "r2": 0.43209290504455566, + "score": -0.5520572896128711 + } + }, + "time": "0:30:03.338423", + "diversity": { + "train": { + "mean_centered_corr": -0.019723476842045784, + "mean_pairwise_disagreement": NaN, + "member_std": 0.059378866106271744 + }, + "val": { + "mean_centered_corr": -0.019546858966350555, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06028679013252258 + }, + "test": { + "mean_centered_corr": -0.0195509921759367, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05971246585249901 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 24 + ], + "head_selection_time": "0:00:00.023549" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/all_four_combined-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/all_four_combined-evaluation/0.toml new file mode 100644 index 000000000..48f767b9b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/all_four_combined-evaluation/0.toml @@ -0,0 +1,74 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 81 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "delivery-eta" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/all_four_combined-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/all_four_combined-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/all_four_combined-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/all_four_combined-evaluation/0/report.json new file mode 100644 index 000000000..33856da09 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/all_four_combined-evaluation/0/report.json @@ -0,0 +1,502 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9009943008422852, + "min_features_per_member": 188.0, + "mean_features_per_member": 198.21875, + "max_features_per_member": 206.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.831817626953125, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8196317439934785, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/all_four_combined-evaluation/_esam_diagnostics/all_four_combined-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5114855052725168, + "mae": 0.4018738567829132, + "r2": 0.5096423625946045, + "score": -0.5114855052725168 + }, + "val": { + "rmse": 0.5560789252392828, + "mae": 0.44038042426109314, + "r2": 0.41688960790634155, + "score": -0.5560789252392828 + }, + "test": { + "rmse": 0.5493995976501785, + "mae": 0.44166451692581177, + "r2": 0.4375477433204651, + "score": -0.5493995976501785 + } + }, + "time": "0:30:03.338423", + "diversity": { + "train": { + "mean_centered_corr": -0.019723476842045784, + "mean_pairwise_disagreement": NaN, + "member_std": 0.059378866106271744 + }, + "val": { + "mean_centered_corr": -0.019546858966350555, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06028679013252258 + }, + "test": { + "mean_centered_corr": -0.0195509921759367, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05971246585249901 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/all_four_combined-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/all_four_combined-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/all_four_combined-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/all_four_combined-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..2023ea3bd --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/all_four_combined-greedy-heads-evaluation/0/report.json @@ -0,0 +1,510 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9009943008422852, + "min_features_per_member": 188.0, + "mean_features_per_member": 198.21875, + "max_features_per_member": 206.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.831817626953125, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8196317439934785, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/all_four_combined-evaluation/_esam_diagnostics/all_four_combined-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5123158821979983, + "mae": 0.40332356095314026, + "r2": 0.5080488920211792, + "score": -0.5123158821979983 + }, + "val": { + "rmse": 0.5558210261849236, + "mae": 0.4417085349559784, + "r2": 0.41743040084838867, + "score": -0.5558210261849236 + }, + "test": { + "rmse": 0.551614874495521, + "mae": 0.4454870820045471, + "r2": 0.433002769947052, + "score": -0.551614874495521 + } + }, + "time": "0:30:03.338423", + "diversity": { + "train": { + "mean_centered_corr": -0.019723476842045784, + "mean_pairwise_disagreement": NaN, + "member_std": 0.059378866106271744 + }, + "val": { + "mean_centered_corr": -0.019546858966350555, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06028679013252258 + }, + "test": { + "mean_centered_corr": -0.0195509921759367, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05971246585249901 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 24, + 30, + 17, + 8, + 12 + ], + "head_selection_time": "0:00:00.146995" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/baseline_plr-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/baseline_plr-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/baseline_plr-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/baseline_plr-best-head-evaluation/0/report.json new file mode 100644 index 000000000..55ac8b264 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/baseline_plr-best-head-evaluation/0/report.json @@ -0,0 +1,168 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5112224822345067, + "mae": 0.40589213371276855, + "r2": 0.5101465582847595, + "score": -0.5112224822345067 + }, + "val": { + "rmse": 0.5583881453279098, + "mae": 0.4439501166343689, + "r2": 0.412036657333374, + "score": -0.5583881453279098 + }, + "test": { + "rmse": 0.5557654209580059, + "mae": 0.4505227506160736, + "r2": 0.42443811893463135, + "score": -0.5557654209580059 + } + }, + "time": "0:07:16.600998", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023548" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/baseline_plr-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/baseline_plr-evaluation/0.toml new file mode 100644 index 000000000..d07370fc8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/baseline_plr-evaluation/0.toml @@ -0,0 +1,33 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/baseline_plr-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/baseline_plr-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/baseline_plr-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/baseline_plr-evaluation/0/report.json new file mode 100644 index 000000000..5bbb360c8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/baseline_plr-evaluation/0/report.json @@ -0,0 +1,164 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5040057771555991, + "mae": 0.39488768577575684, + "r2": 0.5238790512084961, + "score": -0.5040057771555991 + }, + "val": { + "rmse": 0.5558749906446844, + "mae": 0.4398687481880188, + "r2": 0.4173172116279602, + "score": -0.5558749906446844 + }, + "test": { + "rmse": 0.5496162918403876, + "mae": 0.44109272956848145, + "r2": 0.4371039867401123, + "score": -0.5496162918403876 + } + }, + "time": "0:07:16.600998", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/baseline_plr-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/baseline_plr-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/baseline_plr-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/baseline_plr-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..fc04b59cb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/baseline_plr-greedy-heads-evaluation/0/report.json @@ -0,0 +1,181 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5042286809765437, + "mae": 0.39605751633644104, + "r2": 0.5234577655792236, + "score": -0.5042286809765437 + }, + "val": { + "rmse": 0.5553348954063525, + "mae": 0.44016024470329285, + "r2": 0.4184489846229553, + "score": -0.5553348954063525 + }, + "test": { + "rmse": 0.550360417400049, + "mae": 0.44305530190467834, + "r2": 0.43557870388031006, + "score": -0.550360417400049 + } + }, + "time": "0:07:16.600998", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13, + 6, + 10, + 14, + 26, + 27, + 18, + 25, + 6, + 15, + 8, + 17, + 10, + 26 + ], + "head_selection_time": "0:00:00.432507" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/cf_fisd_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/cf_fisd_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/cf_fisd_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/cf_fisd_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..873cd222c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/cf_fisd_only-best-head-evaluation/0/report.json @@ -0,0 +1,449 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 3003, + "metrics": { + "train": { + "rmse": 0.5091186451729569, + "mae": 0.40107491612434387, + "r2": 0.5141700506210327, + "score": -0.5091186451729569 + }, + "val": { + "rmse": 0.5580564462352832, + "mae": 0.4435662627220154, + "r2": 0.4127349853515625, + "score": -0.5580564462352832 + }, + "test": { + "rmse": 0.5528824139554394, + "mae": 0.4462261497974396, + "r2": 0.4303940534591675, + "score": -0.5528824139554394 + } + }, + "time": "0:09:03.165444", + "diversity": { + "train": { + "mean_centered_corr": -0.017171135172247887, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07136442512273788 + }, + "val": { + "mean_centered_corr": -0.01788639836013317, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07444944232702255 + }, + "test": { + "mean_centered_corr": -0.017493220046162605, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07251272350549698 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 27 + ], + "head_selection_time": "0:00:00.023957" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/cf_fisd_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/cf_fisd_only-evaluation/0.toml new file mode 100644 index 000000000..26f1234c9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/cf_fisd_only-evaluation/0.toml @@ -0,0 +1,44 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "delivery-eta" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/cf_fisd_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/cf_fisd_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/cf_fisd_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/cf_fisd_only-evaluation/0/report.json new file mode 100644 index 000000000..fdaec3b4f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/cf_fisd_only-evaluation/0/report.json @@ -0,0 +1,445 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 3003, + "metrics": { + "train": { + "rmse": 0.5000776588212498, + "mae": 0.39380842447280884, + "r2": 0.5312716960906982, + "score": -0.5000776588212498 + }, + "val": { + "rmse": 0.5562691235201772, + "mae": 0.44062381982803345, + "r2": 0.41649067401885986, + "score": -0.5562691235201772 + }, + "test": { + "rmse": 0.5515996385557942, + "mae": 0.4439227879047394, + "r2": 0.4330340623855591, + "score": -0.5515996385557942 + } + }, + "time": "0:09:03.165444", + "diversity": { + "train": { + "mean_centered_corr": -0.017171135172247887, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07136442512273788 + }, + "val": { + "mean_centered_corr": -0.01788639836013317, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07444944232702255 + }, + "test": { + "mean_centered_corr": -0.017493220046162605, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07251272350549698 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/cf_fisd_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/cf_fisd_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/cf_fisd_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/cf_fisd_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..cd4f2f046 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/cf_fisd_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,469 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 3003, + "metrics": { + "train": { + "rmse": 0.5022562202858329, + "mae": 0.39594367146492004, + "r2": 0.5271788239479065, + "score": -0.5022562202858329 + }, + "val": { + "rmse": 0.5559326756199947, + "mae": 0.4413962960243225, + "r2": 0.41719627380371094, + "score": -0.5559326756199947 + }, + "test": { + "rmse": 0.5522105562224877, + "mae": 0.4452051818370819, + "r2": 0.4317775368690491, + "score": -0.5522105562224877 + } + }, + "time": "0:09:03.165444", + "diversity": { + "train": { + "mean_centered_corr": -0.017171135172247887, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07136442512273788 + }, + "val": { + "mean_centered_corr": -0.01788639836013317, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07444944232702255 + }, + "test": { + "mean_centered_corr": -0.017493220046162605, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07251272350549698 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 27, + 10, + 7, + 26, + 25, + 17, + 1, + 11, + 28, + 30, + 20, + 9, + 26, + 23, + 19, + 3, + 10, + 12, + 29, + 26, + 24 + ], + "head_selection_time": "0:00:00.737306" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/esam_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/esam_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/esam_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/esam_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..6e8396567 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/esam_only-best-head-evaluation/0/report.json @@ -0,0 +1,178 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/esam_only-evaluation/_esam_diagnostics/esam_only-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5123514530013489, + "mae": 0.4078504741191864, + "r2": 0.5079806447029114, + "score": -0.5123514530013489 + }, + "val": { + "rmse": 0.5582863288518123, + "mae": 0.4448302686214447, + "r2": 0.41225099563598633, + "score": -0.5582863288518123 + }, + "test": { + "rmse": 0.5560390501031047, + "mae": 0.4517032206058502, + "r2": 0.4238712191581726, + "score": -0.5560390501031047 + } + }, + "time": "0:12:51.556071", + "diversity": { + "train": { + "mean_centered_corr": -0.02027146704494953, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06810352951288223 + }, + "val": { + "mean_centered_corr": -0.02014581859111786, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07032498717308044 + }, + "test": { + "mean_centered_corr": -0.02043480984866619, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0694202110171318 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023277" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/esam_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/esam_only-evaluation/0.toml new file mode 100644 index 000000000..4b93a10d8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/esam_only-evaluation/0.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/esam_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/esam_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/esam_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/esam_only-evaluation/0/report.json new file mode 100644 index 000000000..239380f66 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/esam_only-evaluation/0/report.json @@ -0,0 +1,174 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/esam_only-evaluation/_esam_diagnostics/esam_only-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.504551673775848, + "mae": 0.39544814825057983, + "r2": 0.5228471159934998, + "score": -0.504551673775848 + }, + "val": { + "rmse": 0.5556360918188752, + "mae": 0.439733624458313, + "r2": 0.41781800985336304, + "score": -0.5556360918188752 + }, + "test": { + "rmse": 0.5494649592968895, + "mae": 0.4416595995426178, + "r2": 0.43741387128829956, + "score": -0.5494649592968895 + } + }, + "time": "0:12:51.556071", + "diversity": { + "train": { + "mean_centered_corr": -0.02027146704494953, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06810352951288223 + }, + "val": { + "mean_centered_corr": -0.02014581859111786, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07032498717308044 + }, + "test": { + "mean_centered_corr": -0.02043480984866619, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0694202110171318 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/esam_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/esam_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/esam_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/esam_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..20d61ffa8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/esam_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,190 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/esam_only-evaluation/_esam_diagnostics/esam_only-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5047633843312425, + "mae": 0.3963008522987366, + "r2": 0.5224465727806091, + "score": -0.5047633843312425 + }, + "val": { + "rmse": 0.5551513290724005, + "mae": 0.4403177499771118, + "r2": 0.4188333749771118, + "score": -0.5551513290724005 + }, + "test": { + "rmse": 0.5507488105906536, + "mae": 0.44414353370666504, + "r2": 0.43478184938430786, + "score": -0.5507488105906536 + } + }, + "time": "0:12:51.556071", + "diversity": { + "train": { + "mean_centered_corr": -0.02027146704494953, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06810352951288223 + }, + "val": { + "mean_centered_corr": -0.02014581859111786, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07032498717308044 + }, + "test": { + "mean_centered_corr": -0.02043480984866619, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0694202110171318 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13, + 21, + 10, + 6, + 29, + 12, + 17, + 15, + 8, + 22, + 16, + 21, + 4 + ], + "head_selection_time": "0:00:00.394226" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/mfb_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/mfb_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/mfb_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/mfb_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..296044dd1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/mfb_only-best-head-evaluation/0/report.json @@ -0,0 +1,204 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9009943008422852, + "min_features_per_member": 188.0, + "mean_features_per_member": 198.21875, + "max_features_per_member": 206.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.831817626953125, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8196317439934785, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5157195206907822, + "mae": 0.4108550548553467, + "r2": 0.501490592956543, + "score": -0.5157195206907822 + }, + "val": { + "rmse": 0.5590040394051904, + "mae": 0.4459855556488037, + "r2": 0.41073888540267944, + "score": -0.5590040394051904 + }, + "test": { + "rmse": 0.5557457406296488, + "mae": 0.45139390230178833, + "r2": 0.42447882890701294, + "score": -0.5557457406296488 + } + }, + "time": "0:08:31.551433", + "diversity": { + "train": { + "mean_centered_corr": -0.02001963183283806, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07454843819141388 + }, + "val": { + "mean_centered_corr": -0.020045898854732513, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07650969177484512 + }, + "test": { + "mean_centered_corr": -0.02007085457444191, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0762816071510315 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023499" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/mfb_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/mfb_only-evaluation/0.toml new file mode 100644 index 000000000..8328925cb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/mfb_only-evaluation/0.toml @@ -0,0 +1,46 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/mfb_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/mfb_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/mfb_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/mfb_only-evaluation/0/report.json new file mode 100644 index 000000000..9fedd8bb3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/mfb_only-evaluation/0/report.json @@ -0,0 +1,200 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9009943008422852, + "min_features_per_member": 188.0, + "mean_features_per_member": 198.21875, + "max_features_per_member": 206.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.831817626953125, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8196317439934785, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.505005369677181, + "mae": 0.3960992991924286, + "r2": 0.521988570690155, + "score": -0.505005369677181 + }, + "val": { + "rmse": 0.5560026829864069, + "mae": 0.4405945837497711, + "r2": 0.4170495271682739, + "score": -0.5560026829864069 + }, + "test": { + "rmse": 0.5501201530906976, + "mae": 0.4423939883708954, + "r2": 0.43607139587402344, + "score": -0.5501201530906976 + } + }, + "time": "0:08:31.551433", + "diversity": { + "train": { + "mean_centered_corr": -0.02001963183283806, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07454843819141388 + }, + "val": { + "mean_centered_corr": -0.020045898854732513, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07650969177484512 + }, + "test": { + "mean_centered_corr": -0.02007085457444191, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0762816071510315 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/mfb_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/mfb_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/mfb_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/mfb_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..2a282e7b9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/mfb_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,217 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9009943008422852, + "min_features_per_member": 188.0, + "mean_features_per_member": 198.21875, + "max_features_per_member": 206.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.831817626953125, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8196317439934785, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.505321820939252, + "mae": 0.3968241810798645, + "r2": 0.5213893055915833, + "score": -0.505321820939252 + }, + "val": { + "rmse": 0.5555016895512279, + "mae": 0.4406256079673767, + "r2": 0.418099582195282, + "score": -0.5555016895512279 + }, + "test": { + "rmse": 0.5503428181880856, + "mae": 0.44352859258651733, + "r2": 0.43561482429504395, + "score": -0.5503428181880856 + } + }, + "time": "0:08:31.551433", + "diversity": { + "train": { + "mean_centered_corr": -0.02001963183283806, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07454843819141388 + }, + "val": { + "mean_centered_corr": -0.020045898854732513, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07650969177484512 + }, + "test": { + "mean_centered_corr": -0.02007085457444191, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0762816071510315 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13, + 6, + 25, + 26, + 15, + 23, + 17, + 6, + 4, + 24, + 21, + 12, + 6, + 10 + ], + "head_selection_time": "0:00:00.433229" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/rla_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/rla_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/rla_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/rla_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..69cbaf349 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/rla_only-best-head-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5153977731507096, + "mae": 0.4045717418193817, + "r2": 0.5021123886108398, + "score": -0.5153977731507096 + }, + "val": { + "rmse": 0.5573103773832353, + "mae": 0.44244086742401123, + "r2": 0.41430413722991943, + "score": -0.5573103773832353 + }, + "test": { + "rmse": 0.5532468973961396, + "mae": 0.4463353753089905, + "r2": 0.4296427369117737, + "score": -0.5532468973961396 + } + }, + "time": "0:14:42.930744", + "diversity": { + "train": { + "mean_centered_corr": -0.02064315602183342, + "mean_pairwise_disagreement": NaN, + "member_std": 0.056059885770082474 + }, + "val": { + "mean_centered_corr": -0.020947575569152832, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05900698900222778 + }, + "test": { + "mean_centered_corr": -0.020849451422691345, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05796251446008682 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 31 + ], + "head_selection_time": "0:00:00.023323" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/rla_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/rla_only-evaluation/0.toml new file mode 100644 index 000000000..9dd7c431e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/rla_only-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/rla_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/rla_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/rla_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/rla_only-evaluation/0/report.json new file mode 100644 index 000000000..5f0d2f187 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/rla_only-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5133709914722272, + "mae": 0.4036697745323181, + "r2": 0.5060205459594727, + "score": -0.5133709914722272 + }, + "val": { + "rmse": 0.5556926485294449, + "mae": 0.4411713182926178, + "r2": 0.4176994562149048, + "score": -0.5556926485294449 + }, + "test": { + "rmse": 0.5502675684403805, + "mae": 0.4438808858394623, + "r2": 0.43576914072036743, + "score": -0.5502675684403805 + } + }, + "time": "0:14:42.930744", + "diversity": { + "train": { + "mean_centered_corr": -0.02064315602183342, + "mean_pairwise_disagreement": NaN, + "member_std": 0.056059885770082474 + }, + "val": { + "mean_centered_corr": -0.020947575569152832, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05900698900222778 + }, + "test": { + "mean_centered_corr": -0.020849451422691345, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05796251446008682 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/rla_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/rla_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/rla_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/rla_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..72b0aed82 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/rla_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,184 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5137274534378229, + "mae": 0.40429580211639404, + "r2": 0.5053342580795288, + "score": -0.5137274534378229 + }, + "val": { + "rmse": 0.5553051102575359, + "mae": 0.4410802721977234, + "r2": 0.4185113310813904, + "score": -0.5553051102575359 + }, + "test": { + "rmse": 0.5508335169889497, + "mae": 0.4448656141757965, + "r2": 0.43460792303085327, + "score": -0.5508335169889497 + } + }, + "time": "0:14:42.930744", + "diversity": { + "train": { + "mean_centered_corr": -0.02064315602183342, + "mean_pairwise_disagreement": NaN, + "member_std": 0.056059885770082474 + }, + "val": { + "mean_centered_corr": -0.020947575569152832, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05900698900222778 + }, + "test": { + "mean_centered_corr": -0.020849451422691345, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05796251446008682 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 31, + 16, + 19, + 15, + 11, + 21 + ], + "head_selection_time": "0:00:00.170577" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/all_four_combined-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/all_four_combined-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/all_four_combined-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/all_four_combined-best-head-evaluation/0/report.json new file mode 100644 index 000000000..a15083dfe --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/all_four_combined-best-head-evaluation/0/report.json @@ -0,0 +1,445 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.887499988079071, + "min_features_per_member": 92.0, + "mean_features_per_member": 97.625, + "max_features_per_member": 103.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.399999618530273, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.797659951359852, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/all_four_combined-evaluation/_esam_diagnostics/all_four_combined-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.788873839009288, + "recall": 0.9686263795008256, + "f1-score": 0.8695577986445486, + "support": 84179.0 + }, + "1": { + "precision": 0.5584350443069721, + "recall": 0.13273984579922105, + "f1-score": 0.21449442892463796, + "support": 25162.0 + }, + "accuracy": 0.7762687372531805, + "macro avg": { + "precision": 0.67365444165813, + "recall": 0.5506831126500233, + "f1-score": 0.5420261137845933, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7358443171254596, + "recall": 0.7762687372531805, + "f1-score": 0.7188119255604138, + "support": 109341.0 + }, + "cross-entropy": 0.4989015904618268, + "roc-auc": 0.6823993201798577, + "score": 0.6823993201798577 + }, + "val": { + "0": { + "precision": 0.6940704108113401, + "recall": 0.9134553421832711, + "f1-score": 0.7887927436632258, + "support": 15518.0 + }, + "1": { + "precision": 0.6500781657113079, + "recall": 0.28537115406611, + "f1-score": 0.39662983864557666, + "support": 8743.0 + }, + "accuracy": 0.6871110011953341, + "macro avg": { + "precision": 0.672074288261324, + "recall": 0.5994132481246905, + "f1-score": 0.5927112911544012, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.678216810427614, + "recall": 0.6871110011953341, + "f1-score": 0.6474679722783157, + "support": 24261.0 + }, + "cross-entropy": 0.6244923069797662, + "roc-auc": 0.6479868261151002, + "score": 0.6479868261151002 + }, + "test": { + "0": { + "precision": 0.6674781317049018, + "recall": 0.9559744711027065, + "f1-score": 0.7860926186889547, + "support": 16922.0 + }, + "1": { + "precision": 0.6642631816133393, + "recall": 0.15462079093674605, + "f1-score": 0.25085091899251194, + "support": 9533.0 + }, + "accuracy": 0.6672084672084672, + "macro avg": { + "precision": 0.6658706566591206, + "recall": 0.5552976310197263, + "f1-score": 0.5184717688407333, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6663196316397774, + "recall": 0.6672084672084672, + "f1-score": 0.593219470958613, + "support": 26455.0 + }, + "cross-entropy": 0.65875352245754, + "roc-auc": 0.6047175337399694, + "score": 0.6047175337399694 + } + }, + "time": "0:03:40.866933", + "diversity": { + "train": { + "mean_centered_corr": -0.010312517173588276, + "mean_pairwise_disagreement": 0.02666120948639223, + "member_std": 0.029578423127532005 + }, + "val": { + "mean_centered_corr": -0.012083624489605427, + "mean_pairwise_disagreement": 0.09328301030061521, + "member_std": 0.06631172448396683 + }, + "test": { + "mean_centered_corr": -0.010351695120334625, + "mean_pairwise_disagreement": 0.058351141012431336, + "member_std": 0.06402193009853363 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 22 + ], + "head_selection_time": "0:00:04.786499" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/all_four_combined-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/all_four_combined-evaluation/0.toml new file mode 100644 index 000000000..b8da71126 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/all_four_combined-evaluation/0.toml @@ -0,0 +1,74 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 47 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "ecom-offers" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/all_four_combined-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/all_four_combined-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/all_four_combined-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/all_four_combined-evaluation/0/report.json new file mode 100644 index 000000000..bf4559d41 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/all_four_combined-evaluation/0/report.json @@ -0,0 +1,441 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.887499988079071, + "min_features_per_member": 92.0, + "mean_features_per_member": 97.625, + "max_features_per_member": 103.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.399999618530273, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.797659951359852, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/all_four_combined-evaluation/_esam_diagnostics/all_four_combined-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7843377382851067, + "recall": 0.9814680621057509, + "f1-score": 0.8718993219534074, + "support": 84179.0 + }, + "1": { + "precision": 0.6104868913857678, + "recall": 0.09717033622128607, + "f1-score": 0.16765522679740802, + "support": 25162.0 + }, + "accuracy": 0.7779698374809083, + "macro avg": { + "precision": 0.6974123148354372, + "recall": 0.5393191991635184, + "f1-score": 0.5197772743754077, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7443304673649471, + "recall": 0.7779698374809083, + "f1-score": 0.7098357783392529, + "support": 109341.0 + }, + "cross-entropy": 0.49273880116728924, + "roc-auc": 0.6982248815909875, + "score": 0.6982248815909875 + }, + "val": { + "0": { + "precision": 0.6767916685698625, + "recall": 0.9548266529191907, + "f1-score": 0.7921199647162599, + "support": 15518.0 + }, + "1": { + "precision": 0.7039695945945946, + "recall": 0.19066681916962142, + "f1-score": 0.3000630006300063, + "support": 8743.0 + }, + "accuracy": 0.6794443757470838, + "macro avg": { + "precision": 0.6903806315822285, + "recall": 0.5727467360444061, + "f1-score": 0.5460914826731331, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6865858488276521, + "recall": 0.6794443757470838, + "f1-score": 0.6147961100933624, + "support": 24261.0 + }, + "cross-entropy": 0.6764607634104041, + "roc-auc": 0.6244437709503304, + "score": 0.6244437709503304 + }, + "test": { + "0": { + "precision": 0.657613234183004, + "recall": 0.9772485521805933, + "f1-score": 0.7861846014880316, + "support": 16922.0 + }, + "1": { + "precision": 0.7056574923547401, + "recall": 0.09682156718766391, + "f1-score": 0.17027949451157642, + "support": 9533.0 + }, + "accuracy": 0.65998865998866, + "macro avg": { + "precision": 0.6816353632688721, + "recall": 0.5370350596841286, + "f1-score": 0.478232047999804, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6749258750127587, + "recall": 0.65998865998866, + "f1-score": 0.5642445756023182, + "support": 26455.0 + }, + "cross-entropy": 0.7397410477591574, + "roc-auc": 0.592244305956134, + "score": 0.592244305956134 + } + }, + "time": "0:03:40.866933", + "diversity": { + "train": { + "mean_centered_corr": -0.010312517173588276, + "mean_pairwise_disagreement": 0.02666120948639223, + "member_std": 0.029578423127532005 + }, + "val": { + "mean_centered_corr": -0.012083624489605427, + "mean_pairwise_disagreement": 0.09328301030061521, + "member_std": 0.06631172448396683 + }, + "test": { + "mean_centered_corr": -0.010351695120334625, + "mean_pairwise_disagreement": 0.058351141012431336, + "member_std": 0.06402193009853363 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/all_four_combined-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/all_four_combined-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/all_four_combined-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/all_four_combined-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..ce05d4015 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/all_four_combined-greedy-heads-evaluation/0/report.json @@ -0,0 +1,447 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.887499988079071, + "min_features_per_member": 92.0, + "mean_features_per_member": 97.625, + "max_features_per_member": 103.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.399999618530273, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.797659951359852, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/all_four_combined-evaluation/_esam_diagnostics/all_four_combined-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7909716267830696, + "recall": 0.9643616578956747, + "f1-score": 0.8691029971468489, + "support": 84179.0 + }, + "1": { + "precision": 0.5528394693695037, + "recall": 0.1474048167872188, + "f1-score": 0.2327507765680399, + "support": 25162.0 + }, + "accuracy": 0.7763601942546712, + "macro avg": { + "precision": 0.6719055480762867, + "recall": 0.5558832373414467, + "f1-score": 0.5509268868574444, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.736171676674326, + "recall": 0.7763601942546712, + "f1-score": 0.7226630105525798, + "support": 109341.0 + }, + "cross-entropy": 0.5000760433868655, + "roc-auc": 0.6809180604528166, + "score": 0.6809180604528166 + }, + "val": { + "0": { + "precision": 0.7073092712054166, + "recall": 0.8549426472483568, + "f1-score": 0.7741502552881109, + "support": 15518.0 + }, + "1": { + "precision": 0.5910247093023255, + "recall": 0.37206908383849935, + "f1-score": 0.45665754193865377, + "support": 8743.0 + }, + "accuracy": 0.6809282387370678, + "macro avg": { + "precision": 0.649166990253871, + "recall": 0.613505865543428, + "f1-score": 0.6154038986133823, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6654034996082555, + "recall": 0.6809282387370678, + "f1-score": 0.6597345760986997, + "support": 24261.0 + }, + "cross-entropy": 0.6204112258943162, + "roc-auc": 0.6505096994576862, + "score": 0.6505096994576862 + }, + "test": { + "0": { + "precision": 0.6799947561615102, + "recall": 0.9195721545916559, + "f1-score": 0.7818419333768779, + "support": 16922.0 + }, + "1": { + "precision": 0.6188742649117894, + "recall": 0.2318262876324347, + "f1-score": 0.3373015873015873, + "support": 9533.0 + }, + "accuracy": 0.6717444717444717, + "macro avg": { + "precision": 0.6494345105366498, + "recall": 0.5756992211120453, + "f1-score": 0.5595717603392326, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6579701240283184, + "recall": 0.6717444717444717, + "f1-score": 0.6216528152844286, + "support": 26455.0 + }, + "cross-entropy": 0.6441552092220911, + "roc-auc": 0.6063775930816054, + "score": 0.6063775930816054 + } + }, + "time": "0:03:40.866933", + "diversity": { + "train": { + "mean_centered_corr": -0.010312517173588276, + "mean_pairwise_disagreement": 0.02666120948639223, + "member_std": 0.029578423127532005 + }, + "val": { + "mean_centered_corr": -0.012083624489605427, + "mean_pairwise_disagreement": 0.09328301030061521, + "member_std": 0.06631172448396683 + }, + "test": { + "mean_centered_corr": -0.010351695120334625, + "mean_pairwise_disagreement": 0.058351141012431336, + "member_std": 0.06402193009853363 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 22, + 24, + 24 + ], + "head_selection_time": "0:00:20.530825" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/baseline_plr-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/baseline_plr-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/baseline_plr-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/baseline_plr-best-head-evaluation/0/report.json new file mode 100644 index 000000000..0da12dd14 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/baseline_plr-best-head-evaluation/0/report.json @@ -0,0 +1,219 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7835229427379597, + "recall": 0.9817888071846897, + "f1-score": 0.871522047464133, + "support": 84179.0 + }, + "1": { + "precision": 0.602952602952603, + "recall": 0.09252046737143312, + "f1-score": 0.1604244909209937, + "support": 25162.0 + }, + "accuracy": 0.7771467244674916, + "macro avg": { + "precision": 0.6932377728452814, + "recall": 0.5371546372780615, + "f1-score": 0.5159732691925634, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7419693545168977, + "recall": 0.7771467244674916, + "f1-score": 0.707881357167369, + "support": 109341.0 + }, + "cross-entropy": 0.49822654924283316, + "roc-auc": 0.6867900811541505, + "score": 0.6867900811541505 + }, + "val": { + "0": { + "precision": 0.6877843221759325, + "recall": 0.9255703054517335, + "f1-score": 0.7891541441169199, + "support": 15518.0 + }, + "1": { + "precision": 0.6580817051509769, + "recall": 0.2542605512981814, + "f1-score": 0.36680141902483293, + "support": 8743.0 + }, + "accuracy": 0.6836486542187049, + "macro avg": { + "precision": 0.6729330136634547, + "recall": 0.5899154283749575, + "f1-score": 0.5779777815708764, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6770803124216278, + "recall": 0.6836486542187049, + "f1-score": 0.6369497883409784, + "support": 24261.0 + }, + "cross-entropy": 0.6337455659020594, + "roc-auc": 0.6404461738890127, + "score": 0.6404461738890127 + }, + "test": { + "0": { + "precision": 0.6663787430075683, + "recall": 0.9573927431745657, + "f1-score": 0.785807828491051, + "support": 16922.0 + }, + "1": { + "precision": 0.6635557629491368, + "recall": 0.14916605475715933, + "f1-score": 0.24357656731757452, + "support": 9533.0 + }, + "accuracy": 0.6661500661500661, + "macro avg": { + "precision": 0.6649672529783526, + "recall": 0.5532793989658625, + "f1-score": 0.5146921979043128, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6653614884660062, + "recall": 0.6661500661500661, + "f1-score": 0.5904160079366473, + "support": 26455.0 + }, + "cross-entropy": 0.6685406788276543, + "roc-auc": 0.6034781946000056, + "score": 0.6034781946000056 + } + }, + "time": "0:01:00.788883", + "diversity": { + "train": { + "mean_centered_corr": -0.013087093830108643, + "mean_pairwise_disagreement": 0.025822423398123243, + "member_std": 0.02709772437810898 + }, + "val": { + "mean_centered_corr": -0.004288136959075928, + "mean_pairwise_disagreement": 0.08692423855623854, + "member_std": 0.060995232313871384 + }, + "test": { + "mean_centered_corr": 0.0036114430986344814, + "mean_pairwise_disagreement": 0.05469726437468373, + "member_std": 0.05942297354340553 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 14 + ], + "head_selection_time": "0:00:04.762789" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/baseline_plr-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/baseline_plr-evaluation/0.toml new file mode 100644 index 000000000..bf2cd1131 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/baseline_plr-evaluation/0.toml @@ -0,0 +1,33 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/baseline_plr-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/baseline_plr-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/baseline_plr-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/baseline_plr-evaluation/0/report.json new file mode 100644 index 000000000..9b9138da9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/baseline_plr-evaluation/0/report.json @@ -0,0 +1,215 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7862557522292872, + "recall": 0.9783200085532021, + "f1-score": 0.8718353174078054, + "support": 84179.0 + }, + "1": { + "precision": 0.6031746031746031, + "recall": 0.11024560845719736, + "f1-score": 0.18641846712140048, + "support": 25162.0 + }, + "accuracy": 0.7785551622904492, + "macro avg": { + "precision": 0.6947151777019451, + "recall": 0.5442828085051997, + "f1-score": 0.529126892264603, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7441243662668947, + "recall": 0.7785551622904492, + "f1-score": 0.7141043767093801, + "support": 109341.0 + }, + "cross-entropy": 0.4922807370219411, + "roc-auc": 0.6989988975549913, + "score": 0.6989988975549913 + }, + "val": { + "0": { + "precision": 0.6750500272876114, + "recall": 0.9565021265627014, + "f1-score": 0.7915000266624007, + "support": 15518.0 + }, + "1": { + "precision": 0.7030356357237132, + "recall": 0.1827747912615807, + "f1-score": 0.29012345679012347, + "support": 8743.0 + }, + "accuracy": 0.6776719838423808, + "macro avg": { + "precision": 0.6890428315056623, + "recall": 0.569638458912141, + "f1-score": 0.5408117417262621, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6851352741676592, + "recall": 0.6776719838423808, + "f1-score": 0.6108176413364323, + "support": 24261.0 + }, + "cross-entropy": 0.6872792555776136, + "roc-auc": 0.6227054517511603, + "score": 0.6227054517511603 + }, + "test": { + "0": { + "precision": 0.656363924552227, + "recall": 0.9788441082614349, + "f1-score": 0.7858057782627259, + "support": 16922.0 + }, + "1": { + "precision": 0.7063166529942576, + "recall": 0.09031784328123361, + "f1-score": 0.16015625, + "support": 9533.0 + }, + "accuracy": 0.6586656586656586, + "macro avg": { + "precision": 0.6813402887732423, + "recall": 0.5345809757713342, + "f1-score": 0.47298101413136295, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6743642783695727, + "recall": 0.6586656586656586, + "f1-score": 0.5603543719906198, + "support": 26455.0 + }, + "cross-entropy": 0.756433532699498, + "roc-auc": 0.591151981311678, + "score": 0.591151981311678 + } + }, + "time": "0:01:00.788883", + "diversity": { + "train": { + "mean_centered_corr": -0.013087093830108643, + "mean_pairwise_disagreement": 0.025822423398123243, + "member_std": 0.02709772437810898 + }, + "val": { + "mean_centered_corr": -0.004288136959075928, + "mean_pairwise_disagreement": 0.08692423855623854, + "member_std": 0.060995232313871384 + }, + "test": { + "mean_centered_corr": 0.0036114430986344814, + "mean_pairwise_disagreement": 0.05469726437468373, + "member_std": 0.05942297354340553 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/baseline_plr-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/baseline_plr-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/baseline_plr-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/baseline_plr-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..5f26c2628 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/baseline_plr-greedy-heads-evaluation/0/report.json @@ -0,0 +1,227 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7829606930730526, + "recall": 0.9834281709214887, + "f1-score": 0.8718189046448073, + "support": 84179.0 + }, + "1": { + "precision": 0.6134663341645885, + "recall": 0.08798982592798664, + "f1-score": 0.15390497375829829, + "support": 25162.0 + }, + "accuracy": 0.7773662212710694, + "macro avg": { + "precision": 0.6982135136188206, + "recall": 0.5357089984247376, + "f1-score": 0.5128619392015528, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7439559550621073, + "recall": 0.7773662212710694, + "f1-score": 0.7066096022882681, + "support": 109341.0 + }, + "cross-entropy": 0.4977984361309486, + "roc-auc": 0.6854164410903827, + "score": 0.6854164410903827 + }, + "val": { + "0": { + "precision": 0.6785484611851171, + "recall": 0.9519267946900374, + "f1-score": 0.7923192447972538, + "support": 15518.0 + }, + "1": { + "precision": 0.7005218787635488, + "recall": 0.19958824202218917, + "f1-score": 0.31066405554566495, + "support": 8743.0 + }, + "accuracy": 0.6808045834879024, + "macro avg": { + "precision": 0.689535169974333, + "recall": 0.5757575183561132, + "f1-score": 0.5514916501714594, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6864670791270086, + "recall": 0.6808045834879024, + "f1-score": 0.6187439049668, + "support": 24261.0 + }, + "cross-entropy": 0.6384330519984234, + "roc-auc": 0.6413971749638401, + "score": 0.6413971749638401 + }, + "test": { + "0": { + "precision": 0.659637469489016, + "recall": 0.9741756293582319, + "f1-score": 0.7866294467110443, + "support": 16922.0 + }, + "1": { + "precision": 0.7015027322404371, + "recall": 0.1077310395468373, + "f1-score": 0.18677821223970173, + "support": 9533.0 + }, + "accuracy": 0.661954261954262, + "macro avg": { + "precision": 0.6805701008647266, + "recall": 0.5409533344525346, + "f1-score": 0.486703829475373, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6747235231578611, + "recall": 0.661954261954262, + "f1-score": 0.5704743978274568, + "support": 26455.0 + }, + "cross-entropy": 0.6791357747346614, + "roc-auc": 0.6040305589800323, + "score": 0.6040305589800323 + } + }, + "time": "0:01:00.788883", + "diversity": { + "train": { + "mean_centered_corr": -0.013087093830108643, + "mean_pairwise_disagreement": 0.025822423398123243, + "member_std": 0.02709772437810898 + }, + "val": { + "mean_centered_corr": -0.004288136959075928, + "mean_pairwise_disagreement": 0.08692423855623854, + "member_std": 0.060995232313871384 + }, + "test": { + "mean_centered_corr": 0.0036114430986344814, + "mean_pairwise_disagreement": 0.05469726437468373, + "member_std": 0.05942297354340553 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 14, + 0, + 25, + 25, + 0, + 25, + 0, + 25, + 0 + ], + "head_selection_time": "0:00:48.737044" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/cf_fisd_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/cf_fisd_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/cf_fisd_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/cf_fisd_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..480f3ae3e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/cf_fisd_only-best-head-evaluation/0/report.json @@ -0,0 +1,390 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7849017777184993, + "recall": 0.9787239097637178, + "f1-score": 0.8711623357741415, + "support": 84179.0 + }, + "1": { + "precision": 0.5906285714285714, + "recall": 0.10269453938478658, + "f1-score": 0.17496699055422013, + "support": 25162.0 + }, + "accuracy": 0.7771284330671935, + "macro avg": { + "precision": 0.6877651745735354, + "recall": 0.5407092245742522, + "f1-score": 0.5230646631641808, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7401948295776631, + "recall": 0.7771284330671935, + "f1-score": 0.7109510035527089, + "support": 109341.0 + }, + "cross-entropy": 0.49898712269253065, + "roc-auc": 0.681041160411764, + "score": 0.681041160411764 + }, + "val": { + "0": { + "precision": 0.6882634613505326, + "recall": 0.9077200670189457, + "f1-score": 0.7829035126722989, + "support": 15518.0 + }, + "1": { + "precision": 0.6226613965744401, + "recall": 0.27027336154638, + "f1-score": 0.3769341202743659, + "support": 8743.0 + }, + "accuracy": 0.6780017311734883, + "macro avg": { + "precision": 0.6554624289624864, + "recall": 0.5889967142826629, + "f1-score": 0.5799188164733324, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6646222737516135, + "recall": 0.6780017311734883, + "f1-score": 0.6366032613333134, + "support": 24261.0 + }, + "cross-entropy": 0.630002257594146, + "roc-auc": 0.640920668337369, + "score": 0.640920668337369 + }, + "test": { + "0": { + "precision": 0.668389645318541, + "recall": 0.9399007209549699, + "f1-score": 0.7812269757846653, + "support": 16922.0 + }, + "1": { + "precision": 0.6175253854832644, + "recall": 0.17224378474771845, + "f1-score": 0.2693569553805774, + "support": 9533.0 + }, + "accuracy": 0.6632772632772633, + "macro avg": { + "precision": 0.6429575154009026, + "recall": 0.5560722528513442, + "f1-score": 0.5252919655826214, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6500608232051526, + "recall": 0.6632772632772633, + "f1-score": 0.5967757603428898, + "support": 26455.0 + }, + "cross-entropy": 0.6609475697303346, + "roc-auc": 0.6022998594088651, + "score": 0.6022998594088651 + } + }, + "time": "0:01:10.756545", + "diversity": { + "train": { + "mean_centered_corr": -0.01448837574571371, + "mean_pairwise_disagreement": 0.025696983482570507, + "member_std": 0.02678096853196621 + }, + "val": { + "mean_centered_corr": -0.006191454362124205, + "mean_pairwise_disagreement": 0.0848030690435067, + "member_std": 0.060151606798172 + }, + "test": { + "mean_centered_corr": 2.267197305627633e-05, + "mean_pairwise_disagreement": 0.053483014370111145, + "member_std": 0.05869301036000252 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0 + ], + "head_selection_time": "0:00:04.815166" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/cf_fisd_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/cf_fisd_only-evaluation/0.toml new file mode 100644 index 000000000..6d8c4a5f0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/cf_fisd_only-evaluation/0.toml @@ -0,0 +1,44 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "ecom-offers" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/cf_fisd_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/cf_fisd_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/cf_fisd_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/cf_fisd_only-evaluation/0/report.json new file mode 100644 index 000000000..87a7db82c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/cf_fisd_only-evaluation/0/report.json @@ -0,0 +1,386 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7863576019789309, + "recall": 0.97808241960584, + "f1-score": 0.8718035598945373, + "support": 84179.0 + }, + "1": { + "precision": 0.6021992238033635, + "recall": 0.11100071536443844, + "f1-score": 0.1874496644295302, + "support": 25162.0 + }, + "accuracy": 0.7785460165903001, + "macro avg": { + "precision": 0.6942784128911472, + "recall": 0.5445415674851392, + "f1-score": 0.5296266121620338, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.743978319626898, + "recall": 0.7785460165903001, + "f1-score": 0.7143172307253279, + "support": 109341.0 + }, + "cross-entropy": 0.4923443699583362, + "roc-auc": 0.6988661347453451, + "score": 0.6988661347453451 + }, + "val": { + "0": { + "precision": 0.6754186385147434, + "recall": 0.9565021265627014, + "f1-score": 0.791753347202219, + "support": 15518.0 + }, + "1": { + "precision": 0.7045951859956237, + "recall": 0.18414731785428343, + "f1-score": 0.2919840406238665, + "support": 8743.0 + }, + "accuracy": 0.6781666048390421, + "macro avg": { + "precision": 0.6900069122551835, + "recall": 0.5703247222084924, + "f1-score": 0.5418686939130428, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6859330672120492, + "recall": 0.6781666048390421, + "f1-score": 0.6116501755516467, + "support": 24261.0 + }, + "cross-entropy": 0.6863324565732282, + "roc-auc": 0.6228570653182646, + "score": 0.6228570653182646 + }, + "test": { + "0": { + "precision": 0.6563998890078091, + "recall": 0.9785486349131308, + "f1-score": 0.7857363164013381, + "support": 16922.0 + }, + "1": { + "precision": 0.7043973941368078, + "recall": 0.09073743837197105, + "f1-score": 0.16076572809218473, + "support": 9533.0 + }, + "accuracy": 0.6586278586278587, + "macro avg": { + "precision": 0.6803986415723084, + "recall": 0.534643036642551, + "f1-score": 0.4732510222467614, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6736956824833239, + "recall": 0.6586278586278587, + "f1-score": 0.5605295646209125, + "support": 26455.0 + }, + "cross-entropy": 0.7549517436640245, + "roc-auc": 0.5912083608375949, + "score": 0.5912083608375949 + } + }, + "time": "0:01:10.756545", + "diversity": { + "train": { + "mean_centered_corr": -0.01448837574571371, + "mean_pairwise_disagreement": 0.025696983482570507, + "member_std": 0.02678096853196621 + }, + "val": { + "mean_centered_corr": -0.006191454362124205, + "mean_pairwise_disagreement": 0.0848030690435067, + "member_std": 0.060151606798172 + }, + "test": { + "mean_centered_corr": 2.267197305627633e-05, + "mean_pairwise_disagreement": 0.053483014370111145, + "member_std": 0.05869301036000252 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/cf_fisd_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/cf_fisd_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/cf_fisd_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/cf_fisd_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..1066b9b8d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/cf_fisd_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,391 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7827204839776917, + "recall": 0.9836657598688509, + "f1-score": 0.871763287694308, + "support": 84179.0 + }, + "1": { + "precision": 0.6127851309490284, + "recall": 0.0864796121135045, + "f1-score": 0.15156897572528122, + "support": 25162.0 + }, + "accuracy": 0.777201598668386, + "macro avg": { + "precision": 0.6977528074633601, + "recall": 0.5350726859911776, + "f1-score": 0.5116661317097946, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7436142625885768, + "recall": 0.777201598668386, + "f1-score": 0.7060292146771904, + "support": 109341.0 + }, + "cross-entropy": 0.49797948962611205, + "roc-auc": 0.6847429264219673, + "score": 0.6847429264219673 + }, + "val": { + "0": { + "precision": 0.6772866480778901, + "recall": 0.9548266529191907, + "f1-score": 0.7924588848776575, + "support": 15518.0 + }, + "1": { + "precision": 0.7059563758389261, + "recall": 0.19249685462655838, + "f1-score": 0.3025074143974117, + "support": 8743.0 + }, + "accuracy": 0.6801038704092989, + "macro avg": { + "precision": 0.6916215119584082, + "recall": 0.5736617537728745, + "f1-score": 0.5474831496375345, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6876184328276835, + "recall": 0.6801038704092989, + "f1-score": 0.6158937924903367, + "support": 24261.0 + }, + "cross-entropy": 0.6393924934998982, + "roc-auc": 0.6418041361448852, + "score": 0.6418041361448852 + }, + "test": { + "0": { + "precision": 0.6587311081867847, + "recall": 0.9761848481266989, + "f1-score": 0.7866377770899308, + "support": 16922.0 + }, + "1": { + "precision": 0.7075471698113207, + "recall": 0.1022763033672506, + "f1-score": 0.17871872422326093, + "support": 9533.0 + }, + "accuracy": 0.6612738612738612, + "macro avg": { + "precision": 0.6831391389990527, + "recall": 0.5392305757469747, + "f1-score": 0.48267825065659586, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6763218666621845, + "recall": 0.6612738612738612, + "f1-score": 0.5675755079166946, + "support": 26455.0 + }, + "cross-entropy": 0.6805522467825859, + "roc-auc": 0.6039617691395597, + "score": 0.6039617691395597 + } + }, + "time": "0:01:10.756545", + "diversity": { + "train": { + "mean_centered_corr": -0.01448837574571371, + "mean_pairwise_disagreement": 0.025696983482570507, + "member_std": 0.02678096853196621 + }, + "val": { + "mean_centered_corr": -0.006191454362124205, + "mean_pairwise_disagreement": 0.0848030690435067, + "member_std": 0.060151606798172 + }, + "test": { + "mean_centered_corr": 2.267197305627633e-05, + "mean_pairwise_disagreement": 0.053483014370111145, + "member_std": 0.05869301036000252 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0, + 25 + ], + "head_selection_time": "0:00:15.897517" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/esam_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/esam_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/esam_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/esam_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..da99ccee1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/esam_only-best-head-evaluation/0/report.json @@ -0,0 +1,229 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/esam_only-evaluation/_esam_diagnostics/esam_only-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7843107416636549, + "recall": 0.9796267477636941, + "f1-score": 0.8711553393442882, + "support": 84179.0 + }, + "1": { + "precision": 0.5915694212907835, + "recall": 0.09872029250457039, + "f1-score": 0.1692040461837131, + "support": 25162.0 + }, + "accuracy": 0.7769089362636157, + "macro avg": { + "precision": 0.6879400814772192, + "recall": 0.5391735201341322, + "f1-score": 0.5201796927640007, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7399563174017385, + "recall": 0.7769089362636157, + "f1-score": 0.709619424742205, + "support": 109341.0 + }, + "cross-entropy": 0.4992874898761024, + "roc-auc": 0.680286897888579, + "score": 0.680286897888579 + }, + "val": { + "0": { + "precision": 0.6876976212482366, + "recall": 0.9110065730119861, + "f1-score": 0.7837560637560638, + "support": 15518.0 + }, + "1": { + "precision": 0.6271598272138229, + "recall": 0.2656982729040375, + "f1-score": 0.3732626335663212, + "support": 8743.0 + }, + "accuracy": 0.6784551337537612, + "macro avg": { + "precision": 0.6574287242310297, + "recall": 0.5883524229580118, + "f1-score": 0.5785093486611925, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6658814581369519, + "recall": 0.6784551337537612, + "f1-score": 0.6358254730900188, + "support": 24261.0 + }, + "cross-entropy": 0.6313069278816857, + "roc-auc": 0.6406711951042247, + "score": 0.6406711951042247 + }, + "test": { + "0": { + "precision": 0.6672369189685388, + "recall": 0.9449828625457983, + "f1-score": 0.7821854822930934, + "support": 16922.0 + }, + "1": { + "precision": 0.6259541984732825, + "recall": 0.16343228784223224, + "f1-score": 0.2591914822824821, + "support": 9533.0 + }, + "accuracy": 0.6633528633528634, + "macro avg": { + "precision": 0.6465955587209107, + "recall": 0.5542075751940153, + "f1-score": 0.5206884822877877, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6523607830970105, + "recall": 0.6633528633528634, + "f1-score": 0.5937257657139531, + "support": 26455.0 + }, + "cross-entropy": 0.6637267283308612, + "roc-auc": 0.6023965414622969, + "score": 0.6023965414622969 + } + }, + "time": "0:01:33.770385", + "diversity": { + "train": { + "mean_centered_corr": -0.013189224526286125, + "mean_pairwise_disagreement": 0.02590141569537856, + "member_std": 0.027156289666891098 + }, + "val": { + "mean_centered_corr": -0.003924679942429066, + "mean_pairwise_disagreement": 0.08662631915552772, + "member_std": 0.060857683420181274 + }, + "test": { + "mean_centered_corr": 0.004035224672406912, + "mean_pairwise_disagreement": 0.054524420653452914, + "member_std": 0.059433531016111374 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0 + ], + "head_selection_time": "0:00:04.761324" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/esam_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/esam_only-evaluation/0.toml new file mode 100644 index 000000000..6c0027518 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/esam_only-evaluation/0.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/esam_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/esam_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/esam_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/esam_only-evaluation/0/report.json new file mode 100644 index 000000000..74020b221 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/esam_only-evaluation/0/report.json @@ -0,0 +1,225 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/esam_only-evaluation/_esam_diagnostics/esam_only-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.786244164176397, + "recall": 0.9782962496584658, + "f1-score": 0.8718187592631802, + "support": 84179.0 + }, + "1": { + "precision": 0.6028260869565217, + "recall": 0.1102058659883952, + "f1-score": 0.18634500369598817, + "support": 25162.0 + }, + "accuracy": 0.7785277251900019, + "macro avg": { + "precision": 0.6945351255664594, + "recall": 0.5442510578234305, + "f1-score": 0.5290818814795841, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7440352429208157, + "recall": 0.7785277251900019, + "f1-score": 0.7140747232878215, + "support": 109341.0 + }, + "cross-entropy": 0.49232123937555056, + "roc-auc": 0.6989090994233629, + "score": 0.6989090994233629 + }, + "val": { + "0": { + "precision": 0.6752969013059107, + "recall": 0.956373243974739, + "f1-score": 0.7916255500733431, + "support": 15518.0 + }, + "1": { + "precision": 0.7035901926444834, + "recall": 0.18380418620610775, + "f1-score": 0.29146640065294277, + "support": 8743.0 + }, + "accuracy": 0.6779605127570999, + "macro avg": { + "precision": 0.689443546975197, + "recall": 0.5700887150904234, + "f1-score": 0.541545975363143, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6854930286779539, + "recall": 0.6779605127570999, + "f1-score": 0.611381889738544, + "support": 24261.0 + }, + "cross-entropy": 0.6870363427483223, + "roc-auc": 0.6227207494642631, + "score": 0.6227207494642631 + }, + "test": { + "0": { + "precision": 0.6564519326065411, + "recall": 0.9785486349131308, + "f1-score": 0.7857736019170997, + "support": 16922.0 + }, + "1": { + "precision": 0.7048780487804878, + "recall": 0.09094723591733976, + "f1-score": 0.1611074979095048, + "support": 9533.0 + }, + "accuracy": 0.6587034587034587, + "macro avg": { + "precision": 0.6806649906935145, + "recall": 0.5347479354152352, + "f1-score": 0.4734405499133022, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6739021751121632, + "recall": 0.6587034587034587, + "f1-score": 0.5606765703728017, + "support": 26455.0 + }, + "cross-entropy": 0.7560552620145878, + "roc-auc": 0.5911612394559284, + "score": 0.5911612394559284 + } + }, + "time": "0:01:33.770385", + "diversity": { + "train": { + "mean_centered_corr": -0.013189224526286125, + "mean_pairwise_disagreement": 0.02590141569537856, + "member_std": 0.027156289666891098 + }, + "val": { + "mean_centered_corr": -0.003924679942429066, + "mean_pairwise_disagreement": 0.08662631915552772, + "member_std": 0.060857683420181274 + }, + "test": { + "mean_centered_corr": 0.004035224672406912, + "mean_pairwise_disagreement": 0.054524420653452914, + "member_std": 0.059433531016111374 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/esam_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/esam_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/esam_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/esam_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..67767979d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/esam_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,230 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/esam_only-evaluation/_esam_diagnostics/esam_only-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7830293972986266, + "recall": 0.9834400503688568, + "f1-score": 0.8718661632517654, + "support": 84179.0 + }, + "1": { + "precision": 0.6145977329278407, + "recall": 0.0883475081472061, + "f1-score": 0.1544876472427812, + "support": 25162.0 + }, + "accuracy": 0.7774576782725602, + "macro avg": { + "precision": 0.6988135651132337, + "recall": 0.5358937792580315, + "f1-score": 0.5131769052472732, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7442692109193387, + "recall": 0.7774576782725602, + "f1-score": 0.706780072765872, + "support": 109341.0 + }, + "cross-entropy": 0.4979356402925865, + "roc-auc": 0.6847042861139584, + "score": 0.6847042861139584 + }, + "val": { + "0": { + "precision": 0.6772467413674823, + "recall": 0.95424668127336, + "f1-score": 0.7922317631008747, + "support": 15518.0 + }, + "1": { + "precision": 0.7036727879799666, + "recall": 0.19283998627473406, + "f1-score": 0.30272017236735793, + "support": 8743.0 + }, + "accuracy": 0.6798565599109683, + "macro avg": { + "precision": 0.6904597646737245, + "recall": 0.573543333774047, + "f1-score": 0.5474759677341163, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6867699648757033, + "recall": 0.6798565599109683, + "f1-score": 0.6158251913279412, + "support": 24261.0 + }, + "cross-entropy": 0.6400161363311784, + "roc-auc": 0.6415752785241469, + "score": 0.6415752785241469 + }, + "test": { + "0": { + "precision": 0.6584811640422563, + "recall": 0.9761257534570382, + "f1-score": 0.7864403551788988, + "support": 16922.0 + }, + "1": { + "precision": 0.7051094890510949, + "recall": 0.10133221441309137, + "f1-score": 0.17719893607264056, + "support": 9533.0 + }, + "accuracy": 0.6608958608958609, + "macro avg": { + "precision": 0.6817953265466756, + "recall": 0.5387289839350649, + "f1-score": 0.4818196456257697, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6752835765279588, + "recall": 0.6608958608958609, + "f1-score": 0.5669015742928675, + "support": 26455.0 + }, + "cross-entropy": 0.6818542221067515, + "roc-auc": 0.6040898334194843, + "score": 0.6040898334194843 + } + }, + "time": "0:01:33.770385", + "diversity": { + "train": { + "mean_centered_corr": -0.013189224526286125, + "mean_pairwise_disagreement": 0.02590141569537856, + "member_std": 0.027156289666891098 + }, + "val": { + "mean_centered_corr": -0.003924679942429066, + "mean_pairwise_disagreement": 0.08662631915552772, + "member_std": 0.060857683420181274 + }, + "test": { + "mean_centered_corr": 0.004035224672406912, + "mean_pairwise_disagreement": 0.054524420653452914, + "member_std": 0.059433531016111374 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0, + 25 + ], + "head_selection_time": "0:00:15.755461" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/mfb_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/mfb_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/mfb_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/mfb_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..276526ec8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/mfb_only-best-head-evaluation/0/report.json @@ -0,0 +1,255 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.887499988079071, + "min_features_per_member": 92.0, + "mean_features_per_member": 97.625, + "max_features_per_member": 103.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.399999618530273, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.797659951359852, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7919161090894863, + "recall": 0.9617125411325865, + "f1-score": 0.8685939905690236, + "support": 84179.0 + }, + "1": { + "precision": 0.5468859834106565, + "recall": 0.15459820364041013, + "f1-score": 0.24105344694035633, + "support": 25162.0 + }, + "accuracy": 0.77597607484841, + "macro avg": { + "precision": 0.6694010462500715, + "recall": 0.5581553723864984, + "f1-score": 0.5548237187546899, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7355287701925426, + "recall": 0.77597607484841, + "f1-score": 0.7241817832654089, + "support": 109341.0 + }, + "cross-entropy": 0.5019382114215071, + "roc-auc": 0.6778537614421274, + "score": 0.6778537614421274 + }, + "val": { + "0": { + "precision": 0.7093391589295467, + "recall": 0.8369635262276066, + "f1-score": 0.7678845926451461, + "support": 15518.0 + }, + "1": { + "precision": 0.5748613678373382, + "recall": 0.39128445613633767, + "f1-score": 0.46563223084252076, + "support": 8743.0 + }, + "accuracy": 0.6763529945179506, + "macro avg": { + "precision": 0.6421002633834425, + "recall": 0.6141239911819721, + "f1-score": 0.6167584117438334, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.660877045763594, + "recall": 0.6763529945179506, + "f1-score": 0.6589611187883243, + "support": 24261.0 + }, + "cross-entropy": 0.6229299585114317, + "roc-auc": 0.6501781765294031, + "score": 0.6501781765294031 + }, + "test": { + "0": { + "precision": 0.6828953229398663, + "recall": 0.9059803805696726, + "f1-score": 0.7787767956923702, + "support": 16922.0 + }, + "1": { + "precision": 0.6027465667915106, + "recall": 0.25322563726004405, + "f1-score": 0.3566257940611612, + "support": 9533.0 + }, + "accuracy": 0.6707616707616708, + "macro avg": { + "precision": 0.6428209448656885, + "recall": 0.5796030089148583, + "f1-score": 0.5677012948767657, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6540138981671476, + "recall": 0.6707616707616708, + "f1-score": 0.6266556277260004, + "support": 26455.0 + }, + "cross-entropy": 0.6442264400252198, + "roc-auc": 0.6048261425892079, + "score": 0.6048261425892079 + } + }, + "time": "0:01:07.311309", + "diversity": { + "train": { + "mean_centered_corr": -0.010898238979279995, + "mean_pairwise_disagreement": 0.026737251557793006, + "member_std": 0.030055008828639984 + }, + "val": { + "mean_centered_corr": -0.008359640836715698, + "mean_pairwise_disagreement": 0.09315727751030128, + "member_std": 0.06754488497972488 + }, + "test": { + "mean_centered_corr": -0.005901505704969168, + "mean_pairwise_disagreement": 0.05854524725492467, + "member_std": 0.06570815294981003 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 24 + ], + "head_selection_time": "0:00:04.817110" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/mfb_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/mfb_only-evaluation/0.toml new file mode 100644 index 000000000..54bfca637 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/mfb_only-evaluation/0.toml @@ -0,0 +1,46 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/mfb_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/mfb_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/mfb_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/mfb_only-evaluation/0/report.json new file mode 100644 index 000000000..078b99182 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/mfb_only-evaluation/0/report.json @@ -0,0 +1,251 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.887499988079071, + "min_features_per_member": 92.0, + "mean_features_per_member": 97.625, + "max_features_per_member": 103.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.399999618530273, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.797659951359852, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7843021876334646, + "recall": 0.9816937716057449, + "f1-score": 0.8719664035791161, + "support": 84179.0 + }, + "1": { + "precision": 0.6124245472837022, + "recall": 0.09677291153326445, + "f1-score": 0.16713569908710274, + "support": 25162.0 + }, + "accuracy": 0.77805214878225, + "macro avg": { + "precision": 0.6983633674585834, + "recall": 0.5392333415695046, + "f1-score": 0.5195510513331094, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7447489991087509, + "recall": 0.77805214878225, + "f1-score": 0.7097678670152651, + "support": 109341.0 + }, + "cross-entropy": 0.4927182387508514, + "roc-auc": 0.698241002315497, + "score": 0.698241002315497 + }, + "val": { + "0": { + "precision": 0.6764142335766423, + "recall": 0.9554710658590024, + "f1-score": 0.792082910411881, + "support": 15518.0 + }, + "1": { + "precision": 0.7048269970098249, + "recall": 0.18872240649662586, + "f1-score": 0.2977264525442079, + "support": 8743.0 + }, + "accuracy": 0.6791558468323647, + "macro avg": { + "precision": 0.6906206152932336, + "recall": 0.5720967361778142, + "f1-score": 0.5449046814780444, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6866534154197781, + "recall": 0.6791558468323647, + "f1-score": 0.6139303812029833, + "support": 24261.0 + }, + "cross-entropy": 0.6745751820826075, + "roc-auc": 0.6250090898119413, + "score": 0.6250090898119413 + }, + "test": { + "0": { + "precision": 0.6571337782014617, + "recall": 0.9776622148682189, + "f1-score": 0.7859755807876859, + "support": 16922.0 + }, + "1": { + "precision": 0.7044566067240031, + "recall": 0.09451379418860799, + "f1-score": 0.16666666666666666, + "support": 9533.0 + }, + "accuracy": 0.6594216594216594, + "macro avg": { + "precision": 0.6807951924627325, + "recall": 0.5360880045284134, + "f1-score": 0.4763211237271763, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6741864534728805, + "recall": 0.6594216594216594, + "f1-score": 0.5628090006207732, + "support": 26455.0 + }, + "cross-entropy": 0.7367691279234707, + "roc-auc": 0.5927112610884333, + "score": 0.5927112610884333 + } + }, + "time": "0:01:07.311309", + "diversity": { + "train": { + "mean_centered_corr": -0.010898238979279995, + "mean_pairwise_disagreement": 0.026737251557793006, + "member_std": 0.030055008828639984 + }, + "val": { + "mean_centered_corr": -0.008359640836715698, + "mean_pairwise_disagreement": 0.09315727751030128, + "member_std": 0.06754488497972488 + }, + "test": { + "mean_centered_corr": -0.005901505704969168, + "mean_pairwise_disagreement": 0.05854524725492467, + "member_std": 0.06570815294981003 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/mfb_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/mfb_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/mfb_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/mfb_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..444970f1f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/mfb_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,256 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.887499988079071, + "min_features_per_member": 92.0, + "mean_features_per_member": 97.625, + "max_features_per_member": 103.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.399999618530273, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.797659951359852, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7904454637331882, + "recall": 0.9662742489219401, + "f1-score": 0.8695605693729521, + "support": 84179.0 + }, + "1": { + "precision": 0.5589560354202268, + "recall": 0.14299340275017883, + "f1-score": 0.22772872559258203, + "support": 25162.0 + }, + "accuracy": 0.776817479262125, + "macro avg": { + "precision": 0.6747007495767074, + "recall": 0.5546338258360595, + "f1-score": 0.5486446474827671, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7371741657277673, + "recall": 0.776817479262125, + "f1-score": 0.7218595893818996, + "support": 109341.0 + }, + "cross-entropy": 0.4989347278683794, + "roc-auc": 0.6834597818089504, + "score": 0.6834597818089504 + }, + "val": { + "0": { + "precision": 0.7030905642970632, + "recall": 0.8840056708338704, + "f1-score": 0.7832367466955951, + "support": 15518.0 + }, + "1": { + "precision": 0.6210526315789474, + "recall": 0.3374127873727554, + "f1-score": 0.43726376639739123, + "support": 8743.0 + }, + "accuracy": 0.6870285643625572, + "macro avg": { + "precision": 0.6620715979380053, + "recall": 0.6107092291033129, + "f1-score": 0.6102502565464931, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6735263399965609, + "recall": 0.6870285643625572, + "f1-score": 0.6585575592446575, + "support": 24261.0 + }, + "cross-entropy": 0.6200265526927166, + "roc-auc": 0.6506721330887921, + "score": 0.6506721330887921 + }, + "test": { + "0": { + "precision": 0.6765875561257216, + "recall": 0.9349958633731238, + "f1-score": 0.7850745528072047, + "support": 16922.0 + }, + "1": { + "precision": 0.6416938110749185, + "recall": 0.2066505821881884, + "f1-score": 0.312623978417837, + "support": 9533.0 + }, + "accuracy": 0.6725382725382726, + "macro avg": { + "precision": 0.6591406836003201, + "recall": 0.5708232227806561, + "f1-score": 0.5488492656125209, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6640136732465189, + "recall": 0.6725382725382726, + "f1-score": 0.6148280464509831, + "support": 26455.0 + }, + "cross-entropy": 0.6485088259427312, + "roc-auc": 0.6064667434006787, + "score": 0.6064667434006787 + } + }, + "time": "0:01:07.311309", + "diversity": { + "train": { + "mean_centered_corr": -0.010898238979279995, + "mean_pairwise_disagreement": 0.026737251557793006, + "member_std": 0.030055008828639984 + }, + "val": { + "mean_centered_corr": -0.008359640836715698, + "mean_pairwise_disagreement": 0.09315727751030128, + "member_std": 0.06754488497972488 + }, + "test": { + "mean_centered_corr": -0.005901505704969168, + "mean_pairwise_disagreement": 0.05854524725492467, + "member_std": 0.06570815294981003 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 24, + 22 + ], + "head_selection_time": "0:00:15.861115" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/rla_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/rla_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/rla_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/rla_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..f9259477f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/rla_only-best-head-evaluation/0/report.json @@ -0,0 +1,228 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7796513262723779, + "recall": 0.9881443115266277, + "f1-score": 0.8716030366376939, + "support": 84179.0 + }, + "1": { + "precision": 0.6235382874387023, + "recall": 0.06569430092997378, + "f1-score": 0.11886527882644807, + "support": 25162.0 + }, + "accuracy": 0.7758663264466211, + "macro avg": { + "precision": 0.7015948068555401, + "recall": 0.5269193062283007, + "f1-score": 0.49523415773207097, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7437259525961453, + "recall": 0.7758663264466211, + "f1-score": 0.6983799322025179, + "support": 109341.0 + }, + "cross-entropy": 0.500117320045548, + "roc-auc": 0.682915560586896, + "score": 0.682915560586896 + }, + "val": { + "0": { + "precision": 0.6736205183780367, + "recall": 0.9613352236112901, + "f1-score": 0.7921622769753611, + "support": 15518.0 + }, + "1": { + "precision": 0.7163120567375887, + "recall": 0.17328148232872012, + "f1-score": 0.279056916559219, + "support": 8743.0 + }, + "accuracy": 0.6773422365112732, + "macro avg": { + "precision": 0.6949662875578126, + "recall": 0.5673083529700051, + "f1-score": 0.53560959676729, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6890053796730189, + "recall": 0.6773422365112732, + "f1-score": 0.6072531567363632, + "support": 24261.0 + }, + "cross-entropy": 0.6460301333180669, + "roc-auc": 0.6394530055211662, + "score": 0.6394530055211662 + }, + "test": { + "0": { + "precision": 0.656934306569343, + "recall": 0.9786077295827916, + "f1-score": 0.7861381438404937, + "support": 16922.0 + }, + "1": { + "precision": 0.7097032878909383, + "recall": 0.09283541382565824, + "f1-score": 0.16419294990723562, + "support": 9533.0 + }, + "accuracy": 0.6594216594216594, + "macro avg": { + "precision": 0.6833187972301407, + "recall": 0.535721571704225, + "f1-score": 0.4751655468738647, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6759494908044128, + "recall": 0.6594216594216594, + "f1-score": 0.5620215861475908, + "support": 26455.0 + }, + "cross-entropy": 0.6825101151285496, + "roc-auc": 0.6014026252811646, + "score": 0.6014026252811646 + } + }, + "time": "0:01:49.861501", + "diversity": { + "train": { + "mean_centered_corr": -0.01670769229531288, + "mean_pairwise_disagreement": 0.024668442555119807, + "member_std": 0.02791455015540123 + }, + "val": { + "mean_centered_corr": 0.002592463046312332, + "mean_pairwise_disagreement": 0.06743399402465924, + "member_std": 0.05864376202225685 + }, + "test": { + "mean_centered_corr": 0.009284000843763351, + "mean_pairwise_disagreement": 0.0444435468629017, + "member_std": 0.060093436390161514 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 11 + ], + "head_selection_time": "0:00:04.811023" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/rla_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/rla_only-evaluation/0.toml new file mode 100644 index 000000000..f6aef79c4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/rla_only-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/rla_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/rla_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/rla_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/rla_only-evaluation/0/report.json new file mode 100644 index 000000000..03bee2ffe --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/rla_only-evaluation/0/report.json @@ -0,0 +1,224 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7814041960149158, + "recall": 0.9857803015003742, + "f1-score": 0.871774130006566, + "support": 84179.0 + }, + "1": { + "precision": 0.61939586645469, + "recall": 0.07741832922661156, + "f1-score": 0.13763380082665067, + "support": 25162.0 + }, + "accuracy": 0.7767443136609323, + "macro avg": { + "precision": 0.7004000312348029, + "recall": 0.5315993153634929, + "f1-score": 0.5047039654166083, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7441221738238402, + "recall": 0.7767443136609323, + "f1-score": 0.7028307422304798, + "support": 109341.0 + }, + "cross-entropy": 0.4956439792677601, + "roc-auc": 0.6916789260357137, + "score": 0.6916789260357137 + }, + "val": { + "0": { + "precision": 0.6749659863945578, + "recall": 0.9590797783219487, + "f1-score": 0.7923232538330494, + "support": 15518.0 + }, + "1": { + "precision": 0.7127996381727725, + "recall": 0.18025849250829235, + "f1-score": 0.28774876757348916, + "support": 8743.0 + }, + "accuracy": 0.6784139153373727, + "macro avg": { + "precision": 0.6938828122836651, + "recall": 0.5696691354151205, + "f1-score": 0.5400360107032692, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6886001984013561, + "recall": 0.6784139153373727, + "f1-score": 0.6104884270176941, + "support": 24261.0 + }, + "cross-entropy": 0.6753608510436575, + "roc-auc": 0.6226365549199251, + "score": 0.6226365549199251 + }, + "test": { + "0": { + "precision": 0.6569933396764985, + "recall": 0.9793168656187212, + "f1-score": 0.7864091491481991, + "support": 16922.0 + }, + "1": { + "precision": 0.7156783103168156, + "recall": 0.0924158187349208, + "f1-score": 0.16369379412857674, + "support": 9533.0 + }, + "accuracy": 0.6597240597240597, + "macro avg": { + "precision": 0.6863358249966571, + "recall": 0.535866342176821, + "f1-score": 0.47505147163838796, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6781403374128109, + "recall": 0.6597240597240597, + "f1-score": 0.5620150656327177, + "support": 26455.0 + }, + "cross-entropy": 0.7367195601018909, + "roc-auc": 0.5911320299643263, + "score": 0.5911320299643263 + } + }, + "time": "0:01:49.861501", + "diversity": { + "train": { + "mean_centered_corr": -0.01670769229531288, + "mean_pairwise_disagreement": 0.024668442555119807, + "member_std": 0.02791455015540123 + }, + "val": { + "mean_centered_corr": 0.002592463046312332, + "mean_pairwise_disagreement": 0.06743399402465924, + "member_std": 0.05864376202225685 + }, + "test": { + "mean_centered_corr": 0.009284000843763351, + "mean_pairwise_disagreement": 0.0444435468629017, + "member_std": 0.060093436390161514 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/rla_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/rla_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/rla_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/rla_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..418bc48ef --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/rla_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,228 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7796513262723779, + "recall": 0.9881443115266277, + "f1-score": 0.8716030366376939, + "support": 84179.0 + }, + "1": { + "precision": 0.6235382874387023, + "recall": 0.06569430092997378, + "f1-score": 0.11886527882644807, + "support": 25162.0 + }, + "accuracy": 0.7758663264466211, + "macro avg": { + "precision": 0.7015948068555401, + "recall": 0.5269193062283007, + "f1-score": 0.49523415773207097, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7437259525961453, + "recall": 0.7758663264466211, + "f1-score": 0.6983799322025179, + "support": 109341.0 + }, + "cross-entropy": 0.500117320045548, + "roc-auc": 0.682915560586896, + "score": 0.682915560586896 + }, + "val": { + "0": { + "precision": 0.6736205183780367, + "recall": 0.9613352236112901, + "f1-score": 0.7921622769753611, + "support": 15518.0 + }, + "1": { + "precision": 0.7163120567375887, + "recall": 0.17328148232872012, + "f1-score": 0.279056916559219, + "support": 8743.0 + }, + "accuracy": 0.6773422365112732, + "macro avg": { + "precision": 0.6949662875578126, + "recall": 0.5673083529700051, + "f1-score": 0.53560959676729, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6890053796730189, + "recall": 0.6773422365112732, + "f1-score": 0.6072531567363632, + "support": 24261.0 + }, + "cross-entropy": 0.6460301333180669, + "roc-auc": 0.6394530055211662, + "score": 0.6394530055211662 + }, + "test": { + "0": { + "precision": 0.656934306569343, + "recall": 0.9786077295827916, + "f1-score": 0.7861381438404937, + "support": 16922.0 + }, + "1": { + "precision": 0.7097032878909383, + "recall": 0.09283541382565824, + "f1-score": 0.16419294990723562, + "support": 9533.0 + }, + "accuracy": 0.6594216594216594, + "macro avg": { + "precision": 0.6833187972301407, + "recall": 0.535721571704225, + "f1-score": 0.4751655468738647, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6759494908044128, + "recall": 0.6594216594216594, + "f1-score": 0.5620215861475908, + "support": 26455.0 + }, + "cross-entropy": 0.6825101151285496, + "roc-auc": 0.6014026252811646, + "score": 0.6014026252811646 + } + }, + "time": "0:01:49.861501", + "diversity": { + "train": { + "mean_centered_corr": -0.01670769229531288, + "mean_pairwise_disagreement": 0.024668442555119807, + "member_std": 0.02791455015540123 + }, + "val": { + "mean_centered_corr": 0.002592463046312332, + "mean_pairwise_disagreement": 0.06743399402465924, + "member_std": 0.05864376202225685 + }, + "test": { + "mean_centered_corr": 0.009284000843763351, + "mean_pairwise_disagreement": 0.0444435468629017, + "member_std": 0.060093436390161514 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 11 + ], + "head_selection_time": "0:00:11.087588" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/all_four_combined-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/all_four_combined-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/all_four_combined-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/all_four_combined-best-head-evaluation/0/report.json new file mode 100644 index 000000000..69cc42063 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/all_four_combined-best-head-evaluation/0/report.json @@ -0,0 +1,680 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.901321291923523, + "min_features_per_member": 256.0, + "mean_features_per_member": 268.59375, + "max_features_per_member": 279.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.842281341552734, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8197222809700668, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/all_four_combined-evaluation/_esam_diagnostics/all_four_combined-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9513552811475944, + "recall": 0.9636150815239791, + "f1-score": 0.9574459372022485, + "support": 181971.0 + }, + "1": { + "precision": 0.8344915508449156, + "recall": 0.7882830763418263, + "f1-score": 0.8107294209075565, + "support": 42349.0 + }, + "accuracy": 0.9305144436519258, + "macro avg": { + "precision": 0.8929234159962549, + "recall": 0.8759490789329027, + "f1-score": 0.8840876790549025, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9292927717209353, + "recall": 0.9305144436519258, + "f1-score": 0.9297475699208473, + "support": 224320.0 + }, + "cross-entropy": 0.15873224362284377, + "roc-auc": 0.9722680689598284, + "score": 0.9722680689598284 + }, + "val": { + "0": { + "precision": 0.9436578686709782, + "recall": 0.9674451944328296, + "f1-score": 0.9554034922133081, + "support": 16741.0 + }, + "1": { + "precision": 0.8168067226890756, + "recall": 0.7153370621136297, + "f1-score": 0.7627118644067796, + "support": 3397.0 + }, + "accuracy": 0.9249180653490913, + "macro avg": { + "precision": 0.8802322956800269, + "recall": 0.8413911282732296, + "f1-score": 0.8590576783100439, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9222598478694823, + "recall": 0.9249180653490913, + "f1-score": 0.9228990995398164, + "support": 20138.0 + }, + "cross-entropy": 0.17990073731774503, + "roc-auc": 0.9569685349939213, + "score": 0.9569685349939213 + }, + "test": { + "0": { + "precision": 0.9437623162043353, + "recall": 0.9471362287974443, + "f1-score": 0.9454462624805436, + "support": 13147.0 + }, + "1": { + "precision": 0.7758787487907127, + "recall": 0.764294790343075, + "f1-score": 0.7700432069131061, + "support": 3148.0 + }, + "accuracy": 0.9118134397054312, + "macro avg": { + "precision": 0.859820532497524, + "recall": 0.8557155095702597, + "f1-score": 0.8577447346968249, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9113292097165732, + "recall": 0.9118134397054312, + "f1-score": 0.9115604804046742, + "support": 16295.0 + }, + "cross-entropy": 0.19410702240157, + "roc-auc": 0.9582203301945192, + "score": 0.9582203301945192 + } + }, + "time": "0:14:26.086332", + "diversity": { + "train": { + "mean_centered_corr": 0.09211483597755432, + "mean_pairwise_disagreement": 0.03578823167364594, + "member_std": 0.03491802513599396 + }, + "val": { + "mean_centered_corr": 0.1013437807559967, + "mean_pairwise_disagreement": 0.03309443068632885, + "member_std": 0.03697067126631737 + }, + "test": { + "mean_centered_corr": 0.12410959601402283, + "mean_pairwise_disagreement": 0.0401679220817785, + "member_std": 0.03913382440805435 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 6 + ], + "head_selection_time": "0:00:01.132889" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/all_four_combined-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/all_four_combined-evaluation/0.toml new file mode 100644 index 000000000..5b9249ca9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/all_four_combined-evaluation/0.toml @@ -0,0 +1,75 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 15 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "homesite-insurance" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/all_four_combined-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/all_four_combined-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/all_four_combined-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/all_four_combined-evaluation/0/report.json new file mode 100644 index 000000000..6ef0bc1f3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/all_four_combined-evaluation/0/report.json @@ -0,0 +1,676 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.901321291923523, + "min_features_per_member": 256.0, + "mean_features_per_member": 268.59375, + "max_features_per_member": 279.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.842281341552734, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8197222809700668, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/all_four_combined-evaluation/_esam_diagnostics/all_four_combined-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9439755183585884, + "recall": 0.9814915563468904, + "f1-score": 0.9623680537968715, + "support": 181971.0 + }, + "1": { + "precision": 0.9040920351966284, + "recall": 0.7496989303171268, + "f1-score": 0.8196886375958485, + "support": 42349.0 + }, + "accuracy": 0.9377318116975749, + "macro avg": { + "precision": 0.9240337767776083, + "recall": 0.8655952433320087, + "f1-score": 0.89102834569636, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9364459818552635, + "recall": 0.9377318116975749, + "f1-score": 0.9354318439328508, + "support": 224320.0 + }, + "cross-entropy": 0.14691183931212085, + "roc-auc": 0.9764623137919726, + "score": 0.9764623137919726 + }, + "val": { + "0": { + "precision": 0.934692250754427, + "recall": 0.980586583836091, + "f1-score": 0.957089552238806, + "support": 16741.0 + }, + "1": { + "precision": 0.8737864077669902, + "recall": 0.6623491315866942, + "f1-score": 0.753516409912927, + "support": 3397.0 + }, + "accuracy": 0.9269043599165756, + "macro avg": { + "precision": 0.9042393292607086, + "recall": 0.8214678577113925, + "f1-score": 0.8553029810758666, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.924418283695716, + "recall": 0.9269043599165756, + "f1-score": 0.9227495996873605, + "support": 20138.0 + }, + "cross-entropy": 0.17471226841771598, + "roc-auc": 0.958474253988237, + "score": 0.958474253988237 + }, + "test": { + "0": { + "precision": 0.9308116741687237, + "recall": 0.9752034684718948, + "f1-score": 0.9524906207050258, + "support": 13147.0 + }, + "1": { + "precision": 0.8706862356207854, + "recall": 0.6972681067344345, + "f1-score": 0.7743870171106015, + "support": 3148.0 + }, + "accuracy": 0.9215096655415772, + "macro avg": { + "precision": 0.9007489548947545, + "recall": 0.8362357876031646, + "f1-score": 0.8634388189078137, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9191961552642187, + "recall": 0.9215096655415772, + "f1-score": 0.9180831249016966, + "support": 16295.0 + }, + "cross-entropy": 0.17874492250186327, + "roc-auc": 0.962487069534998, + "score": 0.962487069534998 + } + }, + "time": "0:14:26.086332", + "diversity": { + "train": { + "mean_centered_corr": 0.09211483597755432, + "mean_pairwise_disagreement": 0.03578823167364594, + "member_std": 0.03491802513599396 + }, + "val": { + "mean_centered_corr": 0.1013437807559967, + "mean_pairwise_disagreement": 0.03309443068632885, + "member_std": 0.03697067126631737 + }, + "test": { + "mean_centered_corr": 0.12410959601402283, + "mean_pairwise_disagreement": 0.0401679220817785, + "member_std": 0.03913382440805435 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/all_four_combined-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/all_four_combined-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/all_four_combined-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/all_four_combined-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..5acbc3d79 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/all_four_combined-greedy-heads-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.901321291923523, + "min_features_per_member": 256.0, + "mean_features_per_member": 268.59375, + "max_features_per_member": 279.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.842281341552734, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8197222809700668, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/all_four_combined-evaluation/_esam_diagnostics/all_four_combined-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9442336152219873, + "recall": 0.9817443438789697, + "f1-score": 0.9626236963555881, + "support": 181971.0 + }, + "1": { + "precision": 0.9054100227790433, + "recall": 0.7508559824316985, + "f1-score": 0.8209219171539583, + "support": 42349.0 + }, + "accuracy": 0.9381553138373752, + "macro avg": { + "precision": 0.9248218190005153, + "recall": 0.8663001631553341, + "f1-score": 0.8917728067547732, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9369041737260607, + "recall": 0.9381553138373752, + "f1-score": 0.9358720529559367, + "support": 224320.0 + }, + "cross-entropy": 0.14721044455331747, + "roc-auc": 0.9763538308497102, + "score": 0.9763538308497102 + }, + "val": { + "0": { + "precision": 0.9343600818367811, + "recall": 0.9820799235410071, + "f1-score": 0.9576258846142645, + "support": 16741.0 + }, + "1": { + "precision": 0.8819826907946499, + "recall": 0.6599941124521637, + "f1-score": 0.7550092608183195, + "support": 3397.0 + }, + "accuracy": 0.9277485351077565, + "macro avg": { + "precision": 0.9081713863157155, + "recall": 0.8210370179965853, + "f1-score": 0.8563175727162919, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9255247457870185, + "recall": 0.9277485351077565, + "f1-score": 0.9234472834108269, + "support": 20138.0 + }, + "cross-entropy": 0.17362982513958533, + "roc-auc": 0.9592737644154757, + "score": 0.9592737644154757 + }, + "test": { + "0": { + "precision": 0.9306966618287373, + "recall": 0.975507720392485, + "f1-score": 0.9525754818583578, + "support": 13147.0 + }, + "1": { + "precision": 0.8719681908548708, + "recall": 0.6966327827191868, + "f1-score": 0.7745011478015186, + "support": 3148.0 + }, + "accuracy": 0.9216324025774778, + "macro avg": { + "precision": 0.9013324263418041, + "recall": 0.8360702515558359, + "f1-score": 0.8635383148299383, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.919351020427956, + "recall": 0.9216324025774778, + "f1-score": 0.9181736405812219, + "support": 16295.0 + }, + "cross-entropy": 0.17835468537288718, + "roc-auc": 0.9627526496640615, + "score": 0.9627526496640615 + } + }, + "time": "0:14:26.086332", + "diversity": { + "train": { + "mean_centered_corr": 0.09211483597755432, + "mean_pairwise_disagreement": 0.03578823167364594, + "member_std": 0.03491802513599396 + }, + "val": { + "mean_centered_corr": 0.1013437807559967, + "mean_pairwise_disagreement": 0.03309443068632885, + "member_std": 0.03697067126631737 + }, + "test": { + "mean_centered_corr": 0.12410959601402283, + "mean_pairwise_disagreement": 0.0401679220817785, + "member_std": 0.03913382440805435 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 6, + 31, + 30, + 28, + 8, + 20, + 12, + 30, + 8, + 21, + 15 + ], + "head_selection_time": "0:00:14.217902" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/baseline_plr-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/baseline_plr-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/baseline_plr-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/baseline_plr-best-head-evaluation/0/report.json new file mode 100644 index 000000000..67ae69ad9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/baseline_plr-best-head-evaluation/0/report.json @@ -0,0 +1,262 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9509438009438009, + "recall": 0.9745069269279171, + "f1-score": 0.9625811844722908, + "support": 181971.0 + }, + "1": { + "precision": 0.8774048625792812, + "recall": 0.7839854542019883, + "f1-score": 0.8280686877252491, + "support": 42349.0 + }, + "accuracy": 0.9385386947218259, + "macro avg": { + "precision": 0.9141743317615411, + "recall": 0.8792461905649527, + "f1-score": 0.89532493609877, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9370605069851746, + "recall": 0.9385386947218259, + "f1-score": 0.9371867937592895, + "support": 224320.0 + }, + "cross-entropy": 0.14097183098455576, + "roc-auc": 0.9772540623630491, + "score": 0.9772540623630491 + }, + "val": { + "0": { + "precision": 0.9410365039753932, + "recall": 0.9685801326085658, + "f1-score": 0.9546096785588132, + "support": 16741.0 + }, + "1": { + "precision": 0.8190574475404196, + "recall": 0.7009125699146306, + "f1-score": 0.7553934010152284, + "support": 3397.0 + }, + "accuracy": 0.923428344423478, + "macro avg": { + "precision": 0.8800469757579064, + "recall": 0.8347463512615982, + "f1-score": 0.8550015397870208, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9204603367934683, + "recall": 0.923428344423478, + "f1-score": 0.9210046683882125, + "support": 20138.0 + }, + "cross-entropy": 0.181709302619563, + "roc-auc": 0.9554763470552776, + "score": 0.9554763470552776 + }, + "test": { + "0": { + "precision": 0.9398939744642724, + "recall": 0.9574807940975127, + "f1-score": 0.9486058779201205, + "support": 13147.0 + }, + "1": { + "precision": 0.8073742246726395, + "recall": 0.7442820838627701, + "f1-score": 0.7745454545454545, + "support": 3148.0 + }, + "accuracy": 0.9162933415158024, + "macro avg": { + "precision": 0.873634099568456, + "recall": 0.8508814389801413, + "f1-score": 0.8615756662327876, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9142927365174138, + "recall": 0.9162933415158024, + "f1-score": 0.914979476399197, + "support": 16295.0 + }, + "cross-entropy": 0.18955508005758365, + "roc-auc": 0.959416932798502, + "score": 0.959416932798502 + } + }, + "time": "0:02:48.403690", + "diversity": { + "train": { + "mean_centered_corr": 0.05291927233338356, + "mean_pairwise_disagreement": 0.03400679940235148, + "member_std": 0.03158120810985565 + }, + "val": { + "mean_centered_corr": 0.06288693100214005, + "mean_pairwise_disagreement": 0.03244768356405319, + "member_std": 0.0349886491894722 + }, + "test": { + "mean_centered_corr": 0.07999585568904877, + "mean_pairwise_disagreement": 0.041502439893495925, + "member_std": 0.0369756743311882 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31 + ], + "head_selection_time": "0:00:00.916152" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/baseline_plr-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/baseline_plr-evaluation/0.toml new file mode 100644 index 000000000..d85b237a3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/baseline_plr-evaluation/0.toml @@ -0,0 +1,34 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/baseline_plr-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/baseline_plr-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/baseline_plr-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/baseline_plr-evaluation/0/report.json new file mode 100644 index 000000000..cd297e4bc --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/baseline_plr-evaluation/0/report.json @@ -0,0 +1,258 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9482836172363248, + "recall": 0.984772298882789, + "f1-score": 0.966183574879227, + "support": 181971.0 + }, + "1": { + "precision": 0.9216057939853453, + "recall": 0.769227136414083, + "f1-score": 0.8385502471169687, + "support": 42349.0 + }, + "accuracy": 0.9440798858773182, + "macro avg": { + "precision": 0.9349447056108351, + "recall": 0.8769997176484361, + "f1-score": 0.9023669109980978, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.943247155320955, + "recall": 0.9440798858773182, + "f1-score": 0.9420878910462924, + "support": 224320.0 + }, + "cross-entropy": 0.13271224543526663, + "roc-auc": 0.9813876825616359, + "score": 0.9813876825616359 + }, + "val": { + "0": { + "precision": 0.9351403678606002, + "recall": 0.9809449853652709, + "f1-score": 0.9574951897848522, + "support": 16741.0 + }, + "1": { + "precision": 0.8762126503686457, + "recall": 0.6647041507212246, + "f1-score": 0.7559424171409441, + "support": 3397.0 + }, + "accuracy": 0.9275995630151952, + "macro avg": { + "precision": 0.9056765091146229, + "recall": 0.8228245680432478, + "f1-score": 0.8567188034628981, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9252000830100605, + "recall": 0.9275995630151952, + "f1-score": 0.9234960454472141, + "support": 20138.0 + }, + "cross-entropy": 0.17351272311805546, + "roc-auc": 0.9583121450834431, + "score": 0.9583121450834431 + }, + "test": { + "0": { + "precision": 0.9321849105974706, + "recall": 0.975507720392485, + "f1-score": 0.9533543950938487, + "support": 13147.0 + }, + "1": { + "precision": 0.8730784391013008, + "recall": 0.7036213468869124, + "f1-score": 0.7792436235708003, + "support": 3148.0 + }, + "accuracy": 0.9229825099723842, + "macro avg": { + "precision": 0.9026316748493857, + "recall": 0.8395645336396986, + "f1-score": 0.8662990093323245, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9207662439960627, + "recall": 0.9229825099723842, + "f1-score": 0.9197182669100772, + "support": 16295.0 + }, + "cross-entropy": 0.17696141048282132, + "roc-auc": 0.9626260391126089, + "score": 0.9626260391126089 + } + }, + "time": "0:02:48.403690", + "diversity": { + "train": { + "mean_centered_corr": 0.05291927233338356, + "mean_pairwise_disagreement": 0.03400679940235148, + "member_std": 0.03158120810985565 + }, + "val": { + "mean_centered_corr": 0.06288693100214005, + "mean_pairwise_disagreement": 0.03244768356405319, + "member_std": 0.0349886491894722 + }, + "test": { + "mean_centered_corr": 0.07999585568904877, + "mean_pairwise_disagreement": 0.041502439893495925, + "member_std": 0.0369756743311882 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/baseline_plr-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/baseline_plr-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/baseline_plr-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/baseline_plr-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..0b339388c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/baseline_plr-greedy-heads-evaluation/0/report.json @@ -0,0 +1,275 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9453926712602988, + "recall": 0.9862120887394146, + "f1-score": 0.9653710741556594, + "support": 181971.0 + }, + "1": { + "precision": 0.9272584947234142, + "recall": 0.7552244444969185, + "f1-score": 0.8324462201168646, + "support": 42349.0 + }, + "accuracy": 0.9426043152639088, + "macro avg": { + "precision": 0.9363255829918564, + "recall": 0.8707182666181665, + "f1-score": 0.898908647136262, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9419691502092978, + "recall": 0.9426043152639088, + "f1-score": 0.9402764118710262, + "support": 224320.0 + }, + "cross-entropy": 0.13432360872538882, + "roc-auc": 0.9807020207732833, + "score": 0.9807020207732833 + }, + "val": { + "0": { + "precision": 0.9332728098048116, + "recall": 0.9824980586583836, + "f1-score": 0.9572530190600902, + "support": 16741.0 + }, + "1": { + "precision": 0.8834526650755767, + "recall": 0.6538121872240212, + "f1-score": 0.7514802909829132, + "support": 3397.0 + }, + "accuracy": 0.927053332009137, + "macro avg": { + "precision": 0.9083627374401941, + "recall": 0.8181551229412024, + "f1-score": 0.8543666550215017, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9248688455757318, + "recall": 0.927053332009137, + "f1-score": 0.9225420270411127, + "support": 20138.0 + }, + "cross-entropy": 0.17301902153958362, + "roc-auc": 0.9589474980445032, + "score": 0.9589474980445032 + }, + "test": { + "0": { + "precision": 0.9310144506571781, + "recall": 0.9752034684718948, + "f1-score": 0.952596775391931, + "support": 13147.0 + }, + "1": { + "precision": 0.8708399366085579, + "recall": 0.6982210927573063, + "f1-score": 0.7750352609308886, + "support": 3148.0 + }, + "accuracy": 0.921693771095428, + "macro avg": { + "precision": 0.900927193632868, + "recall": 0.8367122806146006, + "f1-score": 0.8638160181614098, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9193894509502093, + "recall": 0.921693771095428, + "f1-score": 0.9182940047553332, + "support": 16295.0 + }, + "cross-entropy": 0.17739591453615167, + "roc-auc": 0.9625170670540112, + "score": 0.9625170670540112 + } + }, + "time": "0:02:48.403690", + "diversity": { + "train": { + "mean_centered_corr": 0.05291927233338356, + "mean_pairwise_disagreement": 0.03400679940235148, + "member_std": 0.03158120810985565 + }, + "val": { + "mean_centered_corr": 0.06288693100214005, + "mean_pairwise_disagreement": 0.03244768356405319, + "member_std": 0.0349886491894722 + }, + "test": { + "mean_centered_corr": 0.07999585568904877, + "mean_pairwise_disagreement": 0.041502439893495925, + "member_std": 0.0369756743311882 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31, + 6, + 9, + 15, + 28, + 3, + 23, + 26, + 2, + 20, + 13, + 6, + 15, + 5 + ], + "head_selection_time": "0:00:13.989049" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/cf_fisd_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/cf_fisd_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/cf_fisd_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/cf_fisd_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..a89b4d488 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/cf_fisd_only-best-head-evaluation/0/report.json @@ -0,0 +1,621 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2420, + "metrics": { + "train": { + "0": { + "precision": 0.9511390416339356, + "recall": 0.9648020838485253, + "f1-score": 0.9579218455024607, + "support": 181971.0 + }, + "1": { + "precision": 0.8388070970177426, + "recall": 0.7870315709934119, + "f1-score": 0.8120949271477998, + "support": 42349.0 + }, + "accuracy": 0.9312410841654779, + "macro avg": { + "precision": 0.8949730693258391, + "recall": 0.8759168274209685, + "f1-score": 0.8850083863251303, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9299320804956013, + "recall": 0.9312410841654779, + "f1-score": 0.9303914239377251, + "support": 224320.0 + }, + "cross-entropy": 0.15615645016500299, + "roc-auc": 0.9724991426058968, + "score": 0.9724991426058968 + }, + "val": { + "0": { + "precision": 0.9410138248847926, + "recall": 0.9758078967803596, + "f1-score": 0.958095070525791, + "support": 16741.0 + }, + "1": { + "precision": 0.8542116630669546, + "recall": 0.6985575507801001, + "f1-score": 0.768582995951417, + "support": 3397.0 + }, + "accuracy": 0.9290396265766213, + "macro avg": { + "precision": 0.8976127439758737, + "recall": 0.8371827237802298, + "f1-score": 0.863339033238604, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9263715096749804, + "recall": 0.9290396265766213, + "f1-score": 0.9261270241791256, + "support": 20138.0 + }, + "cross-entropy": 0.17528768630111824, + "roc-auc": 0.957813544584969, + "score": 0.957813544584969 + }, + "test": { + "0": { + "precision": 0.9342733348020856, + "recall": 0.9676732334372861, + "f1-score": 0.9506800179345389, + "support": 13147.0 + }, + "1": { + "precision": 0.8412994772218073, + "recall": 0.71569250317662, + "f1-score": 0.7734294541709578, + "support": 3148.0 + }, + "accuracy": 0.9189935563056152, + "macro avg": { + "precision": 0.8877864060119465, + "recall": 0.841682868306953, + "f1-score": 0.8620547360527484, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9163118924171383, + "recall": 0.9189935563056152, + "f1-score": 0.9164373192706695, + "support": 16295.0 + }, + "cross-entropy": 0.1803789882797406, + "roc-auc": 0.961490313471295, + "score": 0.961490313471295 + } + }, + "time": "0:03:46.650441", + "diversity": { + "train": { + "mean_centered_corr": 0.0506281740963459, + "mean_pairwise_disagreement": 0.03327413710540242, + "member_std": 0.03079444356262684 + }, + "val": { + "mean_centered_corr": 0.056434109807014465, + "mean_pairwise_disagreement": 0.030276074921749604, + "member_std": 0.03229342773556709 + }, + "test": { + "mean_centered_corr": 0.06734751909971237, + "mean_pairwise_disagreement": 0.037917825574834946, + "member_std": 0.03452758118510246 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25 + ], + "head_selection_time": "0:00:00.860463" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/cf_fisd_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/cf_fisd_only-evaluation/0.toml new file mode 100644 index 000000000..e655e5952 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/cf_fisd_only-evaluation/0.toml @@ -0,0 +1,45 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "homesite-insurance" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/cf_fisd_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/cf_fisd_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/cf_fisd_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/cf_fisd_only-evaluation/0/report.json new file mode 100644 index 000000000..5caf5c656 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/cf_fisd_only-evaluation/0/report.json @@ -0,0 +1,617 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2420, + "metrics": { + "train": { + "0": { + "precision": 0.9480090078156047, + "recall": 0.9831951244978595, + "f1-score": 0.965281524483674, + "support": 181971.0 + }, + "1": { + "precision": 0.9140890574518893, + "recall": 0.7683062173841176, + "f1-score": 0.8348814533511238, + "support": 42349.0 + }, + "accuracy": 0.942626604850214, + "macro avg": { + "precision": 0.931049032633747, + "recall": 0.8757506709409886, + "f1-score": 0.9000814889173989, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9416053167583962, + "recall": 0.942626604850214, + "f1-score": 0.9406635117679449, + "support": 224320.0 + }, + "cross-entropy": 0.13436786219631347, + "roc-auc": 0.9804692077039372, + "score": 0.9804692077039372 + }, + "val": { + "0": { + "precision": 0.9353075170842825, + "recall": 0.9810644525416642, + "f1-score": 0.957639718958631, + "support": 16741.0 + }, + "1": { + "precision": 0.8770364623739333, + "recall": 0.6655872828966736, + "f1-score": 0.7568200836820084, + "support": 3397.0 + }, + "accuracy": 0.9278478498361307, + "macro avg": { + "precision": 0.9061719897291078, + "recall": 0.8233258677191688, + "f1-score": 0.8572299013203197, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9254780020951546, + "recall": 0.9278478498361307, + "f1-score": 0.9237642446794232, + "support": 20138.0 + }, + "cross-entropy": 0.1724175473381701, + "roc-auc": 0.9589599740471012, + "score": 0.9589599740471012 + }, + "test": { + "0": { + "precision": 0.931104503589818, + "recall": 0.9765726021145509, + "f1-score": 0.9532967032967034, + "support": 13147.0 + }, + "1": { + "precision": 0.8770949720670391, + "recall": 0.6982210927573063, + "f1-score": 0.7775026529890343, + "support": 3148.0 + }, + "accuracy": 0.9227984044185333, + "macro avg": { + "precision": 0.9040997378284286, + "recall": 0.8373968474359286, + "f1-score": 0.8653996781428688, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9206705051096887, + "recall": 0.9227984044185333, + "f1-score": 0.919335385692006, + "support": 16295.0 + }, + "cross-entropy": 0.17668487055650292, + "roc-auc": 0.9628908822909436, + "score": 0.9628908822909436 + } + }, + "time": "0:03:46.650441", + "diversity": { + "train": { + "mean_centered_corr": 0.0506281740963459, + "mean_pairwise_disagreement": 0.03327413710540242, + "member_std": 0.03079444356262684 + }, + "val": { + "mean_centered_corr": 0.056434109807014465, + "mean_pairwise_disagreement": 0.030276074921749604, + "member_std": 0.03229342773556709 + }, + "test": { + "mean_centered_corr": 0.06734751909971237, + "mean_pairwise_disagreement": 0.037917825574834946, + "member_std": 0.03452758118510246 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/cf_fisd_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/cf_fisd_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/cf_fisd_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/cf_fisd_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..a6950ee69 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/cf_fisd_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,639 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2420, + "metrics": { + "train": { + "0": { + "precision": 0.9470743368373016, + "recall": 0.9808101290865028, + "f1-score": 0.9636470639051465, + "support": 181971.0 + }, + "1": { + "precision": 0.9026403100342933, + "recall": 0.7644808614134927, + "f1-score": 0.8278357369336197, + "support": 42349.0 + }, + "accuracy": 0.9399696861626248, + "macro avg": { + "precision": 0.9248573234357975, + "recall": 0.8726454952499978, + "f1-score": 0.8957414004193831, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9386857107625842, + "recall": 0.9399696861626248, + "f1-score": 0.9380074691926055, + "support": 224320.0 + }, + "cross-entropy": 0.13890143673129085, + "roc-auc": 0.9784781707405066, + "score": 0.9784781707405066 + }, + "val": { + "0": { + "precision": 0.9363594890510949, + "recall": 0.9808255181888776, + "f1-score": 0.9580768445313184, + "support": 16741.0 + }, + "1": { + "precision": 0.8766333589546502, + "recall": 0.6714748307329997, + "f1-score": 0.7604600766794466, + "support": 3397.0 + }, + "accuracy": 0.9286423676631245, + "macro avg": { + "precision": 0.9064964240028726, + "recall": 0.8261501744609387, + "f1-score": 0.8592684606053824, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9262845231191442, + "recall": 0.9286423676631245, + "f1-score": 0.9247416493583713, + "support": 20138.0 + }, + "cross-entropy": 0.17179269209939912, + "roc-auc": 0.9594844321379927, + "score": 0.9594844321379927 + }, + "test": { + "0": { + "precision": 0.9321799811005307, + "recall": 0.9754316574123374, + "f1-score": 0.9533154921201308, + "support": 13147.0 + }, + "1": { + "precision": 0.8727344365642238, + "recall": 0.7036213468869124, + "f1-score": 0.7791065775589167, + "support": 3148.0 + }, + "accuracy": 0.9229211414544338, + "macro avg": { + "precision": 0.9024572088323772, + "recall": 0.8395265021496249, + "f1-score": 0.8662110348395238, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9206958096245998, + "recall": 0.9229211414544338, + "f1-score": 0.9196604038698268, + "support": 16295.0 + }, + "cross-entropy": 0.1758390088286691, + "roc-auc": 0.9634382071404677, + "score": 0.9634382071404677 + } + }, + "time": "0:03:46.650441", + "diversity": { + "train": { + "mean_centered_corr": 0.0506281740963459, + "mean_pairwise_disagreement": 0.03327413710540242, + "member_std": 0.03079444356262684 + }, + "val": { + "mean_centered_corr": 0.056434109807014465, + "mean_pairwise_disagreement": 0.030276074921749604, + "member_std": 0.03229342773556709 + }, + "test": { + "mean_centered_corr": 0.06734751909971237, + "mean_pairwise_disagreement": 0.037917825574834946, + "member_std": 0.03452758118510246 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25, + 2, + 0, + 28, + 7, + 26, + 18, + 29, + 24, + 15, + 4, + 7, + 27, + 19, + 31, + 0, + 2, + 28, + 11 + ], + "head_selection_time": "0:00:17.095036" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/esam_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/esam_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/esam_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/esam_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..f143c209a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/esam_only-best-head-evaluation/0/report.json @@ -0,0 +1,272 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/esam_only-evaluation/_esam_diagnostics/esam_only-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9545321160743397, + "recall": 0.9652801820070231, + "f1-score": 0.9598760625261277, + "support": 181971.0 + }, + "1": { + "precision": 0.8432258064516129, + "recall": 0.8024274481097546, + "f1-score": 0.8223208992244311, + "support": 42349.0 + }, + "accuracy": 0.934535485021398, + "macro avg": { + "precision": 0.8988789612629764, + "recall": 0.8838538150583889, + "f1-score": 0.8910984808752793, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9335187828619073, + "recall": 0.934535485021398, + "f1-score": 0.9339072518509157, + "support": 224320.0 + }, + "cross-entropy": 0.1498442336366005, + "roc-auc": 0.9747257011404723, + "score": 0.9747257011404723 + }, + "val": { + "0": { + "precision": 0.9430700273589848, + "recall": 0.9677438623738128, + "f1-score": 0.955247641509434, + "support": 16741.0 + }, + "1": { + "precision": 0.8175059141601893, + "recall": 0.7120989108036503, + "f1-score": 0.7611705475141598, + "support": 3397.0 + }, + "accuracy": 0.9246201211639686, + "macro avg": { + "precision": 0.880287970759587, + "recall": 0.8399213865887316, + "f1-score": 0.8582090945117968, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9218891110546691, + "recall": 0.9246201211639686, + "f1-score": 0.9225095399947877, + "support": 20138.0 + }, + "cross-entropy": 0.18072654373317865, + "roc-auc": 0.9561512908829329, + "score": 0.9561512908829329 + }, + "test": { + "0": { + "precision": 0.938474098263322, + "recall": 0.9618163839659238, + "f1-score": 0.9500018782164457, + "support": 13147.0 + }, + "1": { + "precision": 0.8220489188231124, + "recall": 0.7366581956797967, + "f1-score": 0.7770145753057464, + "support": 3148.0 + }, + "accuracy": 0.9183185026081621, + "macro avg": { + "precision": 0.8802615085432172, + "recall": 0.8492372898228603, + "f1-score": 0.8635082267610961, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9159821396945721, + "recall": 0.9183185026081621, + "f1-score": 0.9165827907931329, + "support": 16295.0 + }, + "cross-entropy": 0.18568286621420937, + "roc-auc": 0.9606910602029306, + "score": 0.9606910602029306 + } + }, + "time": "0:05:02.257608", + "diversity": { + "train": { + "mean_centered_corr": 0.049995824694633484, + "mean_pairwise_disagreement": 0.03397909919872533, + "member_std": 0.031108465045690536 + }, + "val": { + "mean_centered_corr": 0.05969587340950966, + "mean_pairwise_disagreement": 0.03266793800197989, + "member_std": 0.03460051864385605 + }, + "test": { + "mean_centered_corr": 0.07376471161842346, + "mean_pairwise_disagreement": 0.04182474833958566, + "member_std": 0.03703681379556656 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 17 + ], + "head_selection_time": "0:00:00.953794" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/esam_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/esam_only-evaluation/0.toml new file mode 100644 index 000000000..016b99992 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/esam_only-evaluation/0.toml @@ -0,0 +1,44 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/esam_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/esam_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/esam_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/esam_only-evaluation/0/report.json new file mode 100644 index 000000000..b0473609b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/esam_only-evaluation/0/report.json @@ -0,0 +1,268 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/esam_only-evaluation/_esam_diagnostics/esam_only-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9496102021429594, + "recall": 0.9833215182638992, + "f1-score": 0.9661718889206862, + "support": 181971.0 + }, + "1": { + "precision": 0.9154336983476832, + "recall": 0.7757916361661432, + "f1-score": 0.8398476443671873, + "support": 42349.0 + }, + "accuracy": 0.9441422967189729, + "macro avg": { + "precision": 0.9325219502453213, + "recall": 0.8795565772150212, + "f1-score": 0.9030097666439367, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9431580767897758, + "recall": 0.9441422967189729, + "f1-score": 0.9423233447311528, + "support": 224320.0 + }, + "cross-entropy": 0.13329761494760606, + "roc-auc": 0.9810524262916843, + "score": 0.9810524262916843 + }, + "val": { + "0": { + "precision": 0.936472602739726, + "recall": 0.9800489815423212, + "f1-score": 0.9577653892180613, + "support": 16741.0 + }, + "1": { + "precision": 0.8724216959511077, + "recall": 0.6723579629084486, + "f1-score": 0.7594347464671655, + "support": 3397.0 + }, + "accuracy": 0.9281457940212533, + "macro avg": { + "precision": 0.9044471493454169, + "recall": 0.8262034722253849, + "f1-score": 0.8586000678426133, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9256681072406231, + "recall": 0.9281457940212533, + "f1-score": 0.9243097732966793, + "support": 20138.0 + }, + "cross-entropy": 0.1734408298186842, + "roc-auc": 0.9583120395781356, + "score": 0.9583120395781356 + }, + "test": { + "0": { + "precision": 0.9324796274738067, + "recall": 0.9748231535711569, + "f1-score": 0.9531813617939088, + "support": 13147.0 + }, + "1": { + "precision": 0.8702469619756958, + "recall": 0.7052096569250318, + "f1-score": 0.779084049833304, + "support": 3148.0 + }, + "accuracy": 0.922737035900583, + "macro avg": { + "precision": 0.9013632947247513, + "recall": 0.8400164052480943, + "f1-score": 0.8661327058136064, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9204570174101029, + "recall": 0.922737035900583, + "f1-score": 0.9195478338373586, + "support": 16295.0 + }, + "cross-entropy": 0.17694458755574416, + "roc-auc": 0.9627483366901237, + "score": 0.9627483366901237 + } + }, + "time": "0:05:02.257608", + "diversity": { + "train": { + "mean_centered_corr": 0.049995824694633484, + "mean_pairwise_disagreement": 0.03397909919872533, + "member_std": 0.031108465045690536 + }, + "val": { + "mean_centered_corr": 0.05969587340950966, + "mean_pairwise_disagreement": 0.03266793800197989, + "member_std": 0.03460051864385605 + }, + "test": { + "mean_centered_corr": 0.07376471161842346, + "mean_pairwise_disagreement": 0.04182474833958566, + "member_std": 0.03703681379556656 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/esam_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/esam_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/esam_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/esam_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..55f982fc5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/esam_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,290 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/esam_only-evaluation/_esam_diagnostics/esam_only-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9495084683651196, + "recall": 0.9824752295695468, + "f1-score": 0.9657105817533626, + "support": 181971.0 + }, + "1": { + "precision": 0.9114928811301379, + "recall": 0.7755082764646155, + "f1-score": 0.8380199030364889, + "support": 42349.0 + }, + "accuracy": 0.9434022824536377, + "macro avg": { + "precision": 0.9305006747476288, + "recall": 0.8789917530170812, + "f1-score": 0.9018652423949257, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9423315688295711, + "recall": 0.9434022824536377, + "f1-score": 0.9416040707290184, + "support": 224320.0 + }, + "cross-entropy": 0.13410595787789584, + "roc-auc": 0.9804656245919036, + "score": 0.9804656245919036 + }, + "val": { + "0": { + "precision": 0.9362795477903392, + "recall": 0.9795113792485515, + "f1-score": 0.957407677711283, + "support": 16741.0 + }, + "1": { + "precision": 0.8692835365853658, + "recall": 0.6714748307329997, + "f1-score": 0.7576814482644079, + "support": 3397.0 + }, + "accuracy": 0.927549905651008, + "macro avg": { + "precision": 0.9027815421878524, + "recall": 0.8254931049907757, + "f1-score": 0.8575445629878454, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9249782542128591, + "recall": 0.927549905651008, + "f1-score": 0.9237166457601939, + "support": 20138.0 + }, + "cross-entropy": 0.17278545942960094, + "roc-auc": 0.958991133281215, + "score": 0.958991133281215 + }, + "test": { + "0": { + "precision": 0.9319371727748691, + "recall": 0.9748231535711569, + "f1-score": 0.9528978772445073, + "support": 13147.0 + }, + "1": { + "precision": 0.8698387731026347, + "recall": 0.7026683608640406, + "f1-score": 0.7773677736777368, + "support": 3148.0 + }, + "accuracy": 0.9222460877569807, + "macro avg": { + "precision": 0.9008879729387519, + "recall": 0.8387457572175987, + "f1-score": 0.865132825461122, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9199405012702239, + "recall": 0.9222460877569807, + "f1-score": 0.9189875510077357, + "support": 16295.0 + }, + "cross-entropy": 0.1769733127255982, + "roc-auc": 0.9627938464179217, + "score": 0.9627938464179217 + } + }, + "time": "0:05:02.257608", + "diversity": { + "train": { + "mean_centered_corr": 0.049995824694633484, + "mean_pairwise_disagreement": 0.03397909919872533, + "member_std": 0.031108465045690536 + }, + "val": { + "mean_centered_corr": 0.05969587340950966, + "mean_pairwise_disagreement": 0.03266793800197989, + "member_std": 0.03460051864385605 + }, + "test": { + "mean_centered_corr": 0.07376471161842346, + "mean_pairwise_disagreement": 0.04182474833958566, + "member_std": 0.03703681379556656 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 17, + 23, + 5, + 26, + 21, + 24, + 30, + 9, + 28, + 15, + 22, + 12, + 20, + 29, + 23, + 26, + 5, + 21, + 30 + ], + "head_selection_time": "0:00:17.134347" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/mfb_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/mfb_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/mfb_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/mfb_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..490610baf --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/mfb_only-best-head-evaluation/0/report.json @@ -0,0 +1,298 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.901321291923523, + "min_features_per_member": 256.0, + "mean_features_per_member": 268.59375, + "max_features_per_member": 279.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.842281341552734, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8197222809700668, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9518098162802934, + "recall": 0.9742541393958378, + "f1-score": 0.9629012065730121, + "support": 181971.0 + }, + "1": { + "precision": 0.8768984182037942, + "recall": 0.7880469432572198, + "f1-score": 0.8301018568035121, + "support": 42349.0 + }, + "accuracy": 0.9391003922967189, + "macro avg": { + "precision": 0.9143541172420437, + "recall": 0.8811505413265288, + "f1-score": 0.8965015316882621, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9376674179335491, + "recall": 0.9391003922967189, + "f1-score": 0.9378302380308022, + "support": 224320.0 + }, + "cross-entropy": 0.1417006278725686, + "roc-auc": 0.9768342693042886, + "score": 0.9768342693042886 + }, + "val": { + "0": { + "precision": 0.9406342396660676, + "recall": 0.9691774684905322, + "f1-score": 0.9546925566343042, + "support": 16741.0 + }, + "1": { + "precision": 0.8213914849428868, + "recall": 0.6985575507801001, + "f1-score": 0.755011135857461, + "support": 3397.0 + }, + "accuracy": 0.9235276591518522, + "macro avg": { + "precision": 0.8810128623044773, + "recall": 0.8338675096353161, + "f1-score": 0.8548518462458826, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9205196484556869, + "recall": 0.9235276591518522, + "f1-score": 0.9210090832814918, + "support": 20138.0 + }, + "cross-entropy": 0.18275894278340477, + "roc-auc": 0.9560981250001208, + "score": 0.9560981250001208 + }, + "test": { + "0": { + "precision": 0.9386612939332886, + "recall": 0.9567962272761846, + "f1-score": 0.9476420069308422, + "support": 13147.0 + }, + "1": { + "precision": 0.8037318590186593, + "recall": 0.738881829733164, + "f1-score": 0.7699437272426349, + "support": 3148.0 + }, + "accuracy": 0.9146977600490949, + "macro avg": { + "precision": 0.8711965764759739, + "recall": 0.8478390285046743, + "f1-score": 0.8587928670867386, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9125945335091553, + "recall": 0.9146977600490949, + "f1-score": 0.9133128148806136, + "support": 16295.0 + }, + "cross-entropy": 0.1874554754465997, + "roc-auc": 0.9601610718172741, + "score": 0.9601610718172741 + } + }, + "time": "0:03:06.973678", + "diversity": { + "train": { + "mean_centered_corr": 0.08821944147348404, + "mean_pairwise_disagreement": 0.03961999131425153, + "member_std": 0.03912428021430969 + }, + "val": { + "mean_centered_corr": 0.09983313083648682, + "mean_pairwise_disagreement": 0.03728577252442021, + "member_std": 0.04294782876968384 + }, + "test": { + "mean_centered_corr": 0.11663155257701874, + "mean_pairwise_disagreement": 0.04569306832691603, + "member_std": 0.04552939534187317 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31 + ], + "head_selection_time": "0:00:01.054713" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/mfb_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/mfb_only-evaluation/0.toml new file mode 100644 index 000000000..fe5f765a4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/mfb_only-evaluation/0.toml @@ -0,0 +1,47 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/mfb_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/mfb_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/mfb_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/mfb_only-evaluation/0/report.json new file mode 100644 index 000000000..5f7ccee67 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/mfb_only-evaluation/0/report.json @@ -0,0 +1,294 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.901321291923523, + "min_features_per_member": 256.0, + "mean_features_per_member": 268.59375, + "max_features_per_member": 279.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.842281341552734, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8197222809700668, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.946151494708437, + "recall": 0.9865253254639476, + "f1-score": 0.9659167032097862, + "support": 181971.0 + }, + "1": { + "precision": 0.9291001619245894, + "recall": 0.7587428274575551, + "f1-score": 0.8353242431726308, + "support": 42349.0 + }, + "accuracy": 0.9435226462196862, + "macro avg": { + "precision": 0.9376258283165132, + "recall": 0.8726340764607513, + "f1-score": 0.9006204731912084, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9429324019299814, + "recall": 0.9435226462196862, + "f1-score": 0.94126236971249, + "support": 224320.0 + }, + "cross-entropy": 0.13654273925686128, + "roc-auc": 0.980952103177431, + "score": 0.980952103177431 + }, + "val": { + "0": { + "precision": 0.9326650809831238, + "recall": 0.9837524640105131, + "f1-score": 0.9575278351114858, + "support": 16741.0 + }, + "1": { + "precision": 0.8903225806451613, + "recall": 0.6499852811304092, + "f1-score": 0.7514037774374681, + "support": 3397.0 + }, + "accuracy": 0.9274505909226338, + "macro avg": { + "precision": 0.9114938308141426, + "recall": 0.8168688725704611, + "f1-score": 0.854465806274477, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9255224911704286, + "recall": 0.9274505909226338, + "f1-score": 0.9227575786848973, + "support": 20138.0 + }, + "cross-entropy": 0.17552008158175839, + "roc-auc": 0.9580942942079151, + "score": 0.9580942942079151 + }, + "test": { + "0": { + "precision": 0.9282410752998989, + "recall": 0.9770289799954363, + "f1-score": 0.9520103761348897, + "support": 13147.0 + }, + "1": { + "precision": 0.877085877085877, + "recall": 0.684561626429479, + "f1-score": 0.7689562890276539, + "support": 3148.0 + }, + "accuracy": 0.9205277692543725, + "macro avg": { + "precision": 0.902663476192888, + "recall": 0.8307953032124576, + "f1-score": 0.8604833325812717, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9183585000327775, + "recall": 0.9205277692543725, + "f1-score": 0.9166465058548297, + "support": 16295.0 + }, + "cross-entropy": 0.1804777281050399, + "roc-auc": 0.9616652728230258, + "score": 0.9616652728230258 + } + }, + "time": "0:03:06.973678", + "diversity": { + "train": { + "mean_centered_corr": 0.08821944147348404, + "mean_pairwise_disagreement": 0.03961999131425153, + "member_std": 0.03912428021430969 + }, + "val": { + "mean_centered_corr": 0.09983313083648682, + "mean_pairwise_disagreement": 0.03728577252442021, + "member_std": 0.04294782876968384 + }, + "test": { + "mean_centered_corr": 0.11663155257701874, + "mean_pairwise_disagreement": 0.04569306832691603, + "member_std": 0.04552939534187317 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/mfb_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/mfb_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/mfb_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/mfb_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..334ff2f4b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/mfb_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,307 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.901321291923523, + "min_features_per_member": 256.0, + "mean_features_per_member": 268.59375, + "max_features_per_member": 279.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.842281341552734, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8197222809700668, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.946386196568126, + "recall": 0.9862395656450753, + "f1-score": 0.9659019657970156, + "support": 181971.0 + }, + "1": { + "precision": 0.9278094908608661, + "recall": 0.7599234928805875, + "f1-score": 0.8355163237489452, + "support": 42349.0 + }, + "accuracy": 0.9435137303851641, + "macro avg": { + "precision": 0.937097843714496, + "recall": 0.8730815292628313, + "f1-score": 0.9007091447729805, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9428791311704943, + "recall": 0.9435137303851641, + "f1-score": 0.9412866771241655, + "support": 224320.0 + }, + "cross-entropy": 0.13649569775840315, + "roc-auc": 0.9805235149654821, + "score": 0.9805235149654821 + }, + "val": { + "0": { + "precision": 0.9322954995754317, + "recall": 0.9837524640105131, + "f1-score": 0.9573330233098878, + "support": 16741.0 + }, + "1": { + "precision": 0.8900121310149616, + "recall": 0.6479246393876951, + "f1-score": 0.7499148211243611, + "support": 3397.0 + }, + "accuracy": 0.927102989373324, + "macro avg": { + "precision": 0.9111538152951966, + "recall": 0.815838551699104, + "f1-score": 0.8536239222171245, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9251628844696655, + "recall": 0.927102989373324, + "f1-score": 0.9223444627366316, + "support": 20138.0 + }, + "cross-entropy": 0.17444519969608893, + "roc-auc": 0.9587921238951639, + "score": 0.9587921238951639 + }, + "test": { + "0": { + "precision": 0.9292688218702538, + "recall": 0.9773332319160265, + "f1-score": 0.9526951879587752, + "support": 13147.0 + }, + "1": { + "precision": 0.8792544570502431, + "recall": 0.6893265565438373, + "f1-score": 0.7727920227920227, + "support": 3148.0 + }, + "accuracy": 0.921693771095428, + "macro avg": { + "precision": 0.9042616394602485, + "recall": 0.8333298942299319, + "f1-score": 0.862743605375399, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9196066420326721, + "recall": 0.921693771095428, + "f1-score": 0.9179400382843391, + "support": 16295.0 + }, + "cross-entropy": 0.1794710390286192, + "roc-auc": 0.9622146031450255, + "score": 0.9622146031450255 + } + }, + "time": "0:03:06.973678", + "diversity": { + "train": { + "mean_centered_corr": 0.08821944147348404, + "mean_pairwise_disagreement": 0.03961999131425153, + "member_std": 0.03912428021430969 + }, + "val": { + "mean_centered_corr": 0.09983313083648682, + "mean_pairwise_disagreement": 0.03728577252442021, + "member_std": 0.04294782876968384 + }, + "test": { + "mean_centered_corr": 0.11663155257701874, + "mean_pairwise_disagreement": 0.04569306832691603, + "member_std": 0.04552939534187317 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31, + 7, + 23, + 22, + 15, + 5, + 2, + 30, + 7, + 28 + ], + "head_selection_time": "0:00:10.953573" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/rla_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/rla_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/rla_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/rla_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..6416ea4eb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/rla_only-best-head-evaluation/0/report.json @@ -0,0 +1,275 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9470929244882358, + "recall": 0.9803869847393266, + "f1-score": 0.9634524044186542, + "support": 181971.0 + }, + "1": { + "precision": 0.9007287494437027, + "recall": 0.7646697678811778, + "f1-score": 0.8271414158184442, + "support": 42349.0 + }, + "accuracy": 0.9396620898716119, + "macro avg": { + "precision": 0.9239108369659692, + "recall": 0.8725283763102523, + "f1-score": 0.8952969101185493, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9383399089347366, + "recall": 0.9396620898716119, + "f1-score": 0.9377184794176276, + "support": 224320.0 + }, + "cross-entropy": 0.1374541129990154, + "roc-auc": 0.9783636099059336, + "score": 0.9783636099059336 + }, + "val": { + "0": { + "precision": 0.9391985260248733, + "recall": 0.9743742906636401, + "f1-score": 0.9564631035797003, + "support": 16741.0 + }, + "1": { + "precision": 0.8451263537906137, + "recall": 0.6891374742419782, + "f1-score": 0.7592022052862007, + "support": 3397.0 + }, + "accuracy": 0.9262588141821432, + "macro avg": { + "precision": 0.8921624399077435, + "recall": 0.8317558824528092, + "f1-score": 0.8578326544329505, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9233298613570919, + "recall": 0.9262588141821432, + "f1-score": 0.9231879386426154, + "support": 20138.0 + }, + "cross-entropy": 0.18034048370527975, + "roc-auc": 0.9557610707114682, + "score": 0.9557610707114682 + }, + "test": { + "0": { + "precision": 0.9347106830380119, + "recall": 0.9669886666159581, + "f1-score": 0.9505757439808584, + "support": 13147.0 + }, + "1": { + "precision": 0.8389012620638456, + "recall": 0.7179161372299873, + "f1-score": 0.7737076343717905, + "support": 3148.0 + }, + "accuracy": 0.9188708192697146, + "macro avg": { + "precision": 0.8868059725509287, + "recall": 0.8424524019229727, + "f1-score": 0.8621416891763245, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.916201443564144, + "recall": 0.9188708192697146, + "f1-score": 0.9164069309063357, + "support": 16295.0 + }, + "cross-entropy": 0.18574135097736744, + "roc-auc": 0.9604946447119459, + "score": 0.9604946447119459 + } + }, + "time": "0:07:11.953824", + "diversity": { + "train": { + "mean_centered_corr": 0.05171258747577667, + "mean_pairwise_disagreement": 0.03286417948437716, + "member_std": 0.03040449135005474 + }, + "val": { + "mean_centered_corr": 0.061806585639715195, + "mean_pairwise_disagreement": 0.03182976975001522, + "member_std": 0.0331539586186409 + }, + "test": { + "mean_centered_corr": 0.0732795000076294, + "mean_pairwise_disagreement": 0.03959877856852983, + "member_std": 0.03512473404407501 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31 + ], + "head_selection_time": "0:00:00.945023" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/rla_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/rla_only-evaluation/0.toml new file mode 100644 index 000000000..d6037d45d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/rla_only-evaluation/0.toml @@ -0,0 +1,41 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/rla_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/rla_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/rla_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/rla_only-evaluation/0/report.json new file mode 100644 index 000000000..f1b6781b1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/rla_only-evaluation/0/report.json @@ -0,0 +1,271 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9493414440624174, + "recall": 0.9862725379318683, + "f1-score": 0.9674546723770353, + "support": 181971.0 + }, + "1": { + "precision": 0.9291749362064077, + "recall": 0.7738553448723701, + "f1-score": 0.8444324198971901, + "support": 42349.0 + }, + "accuracy": 0.9461706490727532, + "macro avg": { + "precision": 0.9392581901344126, + "recall": 0.8800639414021192, + "f1-score": 0.9059435461371127, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9455342425592338, + "recall": 0.9461706490727532, + "f1-score": 0.9442295057834682, + "support": 224320.0 + }, + "cross-entropy": 0.1285332583359511, + "roc-auc": 0.9826472071386247, + "score": 0.9826472071386247 + }, + "val": { + "0": { + "precision": 0.9352083807788658, + "recall": 0.9811839197180575, + "f1-score": 0.9576446582130884, + "support": 16741.0 + }, + "1": { + "precision": 0.8776223776223776, + "recall": 0.6649985281130409, + "f1-score": 0.7566571763523698, + "support": 3397.0 + }, + "accuracy": 0.9278478498361307, + "macro avg": { + "precision": 0.9064153792006218, + "recall": 0.8230912239155492, + "f1-score": 0.8571509172827292, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9254944244414645, + "recall": 0.9278478498361307, + "f1-score": 0.9237408705538938, + "support": 20138.0 + }, + "cross-entropy": 0.173981665844509, + "roc-auc": 0.9583231528038466, + "score": 0.9583231528038466 + }, + "test": { + "0": { + "precision": 0.9317207815791385, + "recall": 0.9756598463527801, + "f1-score": 0.9531842163929554, + "support": 13147.0 + }, + "1": { + "precision": 0.8734177215189873, + "recall": 0.7013977128335451, + "f1-score": 0.7780126849894292, + "support": 3148.0 + }, + "accuracy": 0.9226756673826327, + "macro avg": { + "precision": 0.9025692515490629, + "recall": 0.8385287795931626, + "f1-score": 0.8655984506911922, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9204573245021606, + "recall": 0.9226756673826327, + "f1-score": 0.9193431620291443, + "support": 16295.0 + }, + "cross-entropy": 0.17786928687791626, + "roc-auc": 0.96242593886798, + "score": 0.96242593886798 + } + }, + "time": "0:07:11.953824", + "diversity": { + "train": { + "mean_centered_corr": 0.05171258747577667, + "mean_pairwise_disagreement": 0.03286417948437716, + "member_std": 0.03040449135005474 + }, + "val": { + "mean_centered_corr": 0.061806585639715195, + "mean_pairwise_disagreement": 0.03182976975001522, + "member_std": 0.0331539586186409 + }, + "test": { + "mean_centered_corr": 0.0732795000076294, + "mean_pairwise_disagreement": 0.03959877856852983, + "member_std": 0.03512473404407501 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/rla_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/rla_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/rla_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/rla_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..427f43a81 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/rla_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,285 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.946255084192114, + "recall": 0.9869869374790489, + "f1-score": 0.9661919171541927, + "support": 181971.0 + }, + "1": { + "precision": 0.931394136052845, + "recall": 0.7591206403929255, + "f1-score": 0.836479542054251, + "support": 42349.0 + }, + "accuracy": 0.9439684379457918, + "macro avg": { + "precision": 0.9388246101224795, + "recall": 0.8730537889359872, + "f1-score": 0.9013357296042219, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9434495104904829, + "recall": 0.9439684379457918, + "f1-score": 0.9417037334295696, + "support": 224320.0 + }, + "cross-entropy": 0.1317045399668242, + "roc-auc": 0.9813354497094697, + "score": 0.9813354497094697 + }, + "val": { + "0": { + "precision": 0.9333144315282109, + "recall": 0.9831551281285467, + "f1-score": 0.9575866883872469, + "support": 16741.0 + }, + "1": { + "precision": 0.8873351977626848, + "recall": 0.6538121872240212, + "f1-score": 0.7528813559322034, + "support": 3397.0 + }, + "accuracy": 0.9275995630151952, + "macro avg": { + "precision": 0.9103248146454479, + "recall": 0.818483657676284, + "f1-score": 0.8552340221597252, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.925558375460007, + "recall": 0.9275995630151952, + "f1-score": 0.9230557511367858, + "support": 20138.0 + }, + "cross-entropy": 0.1731441036701711, + "roc-auc": 0.959029757015826, + "score": 0.959029757015826 + }, + "test": { + "0": { + "precision": 0.9292688218702538, + "recall": 0.9773332319160265, + "f1-score": 0.9526951879587752, + "support": 13147.0 + }, + "1": { + "precision": 0.8792544570502431, + "recall": 0.6893265565438373, + "f1-score": 0.7727920227920227, + "support": 3148.0 + }, + "accuracy": 0.921693771095428, + "macro avg": { + "precision": 0.9042616394602485, + "recall": 0.8333298942299319, + "f1-score": 0.862743605375399, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9196066420326721, + "recall": 0.921693771095428, + "f1-score": 0.9179400382843391, + "support": 16295.0 + }, + "cross-entropy": 0.17812450933232857, + "roc-auc": 0.9624579176971494, + "score": 0.9624579176971494 + } + }, + "time": "0:07:11.953824", + "diversity": { + "train": { + "mean_centered_corr": 0.05171258747577667, + "mean_pairwise_disagreement": 0.03286417948437716, + "member_std": 0.03040449135005474 + }, + "val": { + "mean_centered_corr": 0.061806585639715195, + "mean_pairwise_disagreement": 0.03182976975001522, + "member_std": 0.0331539586186409 + }, + "test": { + "mean_centered_corr": 0.0732795000076294, + "mean_pairwise_disagreement": 0.03959877856852983, + "member_std": 0.03512473404407501 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31, + 22, + 6, + 20, + 26, + 21, + 24, + 7, + 22, + 13, + 5 + ], + "head_selection_time": "0:00:11.473881" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/all_four_combined-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/all_four_combined-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/all_four_combined-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/all_four_combined-best-head-evaluation/0/report.json new file mode 100644 index 000000000..8d841fedb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/all_four_combined-best-head-evaluation/0/report.json @@ -0,0 +1,702 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/all_four_combined-evaluation/_esam_diagnostics/all_four_combined-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.238513825213292, + "mae": 0.12740404903888702, + "r2": 0.6365530490875244, + "score": -0.238513825213292 + }, + "val": { + "rmse": 0.2376718937442043, + "mae": 0.13975045084953308, + "r2": 0.6226825714111328, + "score": -0.2376718937442043 + }, + "test": { + "rmse": 0.2539783962159738, + "mae": 0.17701080441474915, + "r2": 0.5592085719108582, + "score": -0.2539783962159738 + } + }, + "time": "0:03:30.033690", + "diversity": { + "train": { + "mean_centered_corr": -0.018900908529758453, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04601670801639557 + }, + "val": { + "mean_centered_corr": -0.01799248345196247, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05731586739420891 + }, + "test": { + "mean_centered_corr": -0.019703488796949387, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07458258420228958 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25 + ], + "head_selection_time": "0:00:00.015071" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/all_four_combined-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/all_four_combined-evaluation/0.toml new file mode 100644 index 000000000..aaa491b98 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/all_four_combined-evaluation/0.toml @@ -0,0 +1,75 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/all_four_combined-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/all_four_combined-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/all_four_combined-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/all_four_combined-evaluation/0/report.json new file mode 100644 index 000000000..ae0cfb53b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/all_four_combined-evaluation/0/report.json @@ -0,0 +1,698 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/all_four_combined-evaluation/_esam_diagnostics/all_four_combined-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2331321585964892, + "mae": 0.12105797231197357, + "r2": 0.6527692079544067, + "score": -0.2331321585964892 + }, + "val": { + "rmse": 0.2348792849855834, + "mae": 0.12763135135173798, + "r2": 0.6314972639083862, + "score": -0.2348792849855834 + }, + "test": { + "rmse": 0.2326272249658597, + "mae": 0.1447165459394455, + "r2": 0.6302052736282349, + "score": -0.2326272249658597 + } + }, + "time": "0:03:30.033690", + "diversity": { + "train": { + "mean_centered_corr": -0.018900908529758453, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04601670801639557 + }, + "val": { + "mean_centered_corr": -0.01799248345196247, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05731586739420891 + }, + "test": { + "mean_centered_corr": -0.019703488796949387, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07458258420228958 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/all_four_combined-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/all_four_combined-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/all_four_combined-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/all_four_combined-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..0f6ab25ae --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/all_four_combined-greedy-heads-evaluation/0/report.json @@ -0,0 +1,710 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/all_four_combined-evaluation/_esam_diagnostics/all_four_combined-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23288428364808966, + "mae": 0.12141954898834229, + "r2": 0.6535071730613708, + "score": -0.23288428364808966 + }, + "val": { + "rmse": 0.23335850745168515, + "mae": 0.1262979656457901, + "r2": 0.6362537145614624, + "score": -0.23335850745168515 + }, + "test": { + "rmse": 0.23263890687901315, + "mae": 0.14496733248233795, + "r2": 0.6301681995391846, + "score": -0.23263890687901315 + } + }, + "time": "0:03:30.033690", + "diversity": { + "train": { + "mean_centered_corr": -0.018900908529758453, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04601670801639557 + }, + "val": { + "mean_centered_corr": -0.01799248345196247, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05731586739420891 + }, + "test": { + "mean_centered_corr": -0.019703488796949387, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07458258420228958 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25, + 10, + 19, + 8, + 12, + 28, + 21, + 24, + 20 + ], + "head_selection_time": "0:00:00.144155" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/baseline_plr-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/baseline_plr-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/baseline_plr-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/baseline_plr-best-head-evaluation/0/report.json new file mode 100644 index 000000000..546302ba0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/baseline_plr-best-head-evaluation/0/report.json @@ -0,0 +1,190 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22658863410177663, + "mae": 0.12739618122577667, + "r2": 0.6719876527786255, + "score": -0.22658863410177663 + }, + "val": { + "rmse": 0.23955454428788905, + "mae": 0.13915589451789856, + "r2": 0.6166812181472778, + "score": -0.23955454428788905 + }, + "test": { + "rmse": 0.26588264760298186, + "mae": 0.1863766759634018, + "r2": 0.5169194340705872, + "score": -0.26588264760298186 + } + }, + "time": "0:00:48.800887", + "diversity": { + "train": { + "mean_centered_corr": -0.018623020499944687, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05047355964779854 + }, + "val": { + "mean_centered_corr": -0.01870153285562992, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06864207983016968 + }, + "test": { + "mean_centered_corr": -0.019026711583137512, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09529124945402145 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015244" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/baseline_plr-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/baseline_plr-evaluation/0.toml new file mode 100644 index 000000000..a68e82651 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/baseline_plr-evaluation/0.toml @@ -0,0 +1,34 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/baseline_plr-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/baseline_plr-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/baseline_plr-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/baseline_plr-evaluation/0/report.json new file mode 100644 index 000000000..c4a6eed6f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/baseline_plr-evaluation/0/report.json @@ -0,0 +1,186 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22224988999004666, + "mae": 0.11853151023387909, + "r2": 0.6844290494918823, + "score": -0.22224988999004666 + }, + "val": { + "rmse": 0.23452539387100468, + "mae": 0.12745676934719086, + "r2": 0.6326068639755249, + "score": -0.23452539387100468 + }, + "test": { + "rmse": 0.23846659721529556, + "mae": 0.15488465130329132, + "r2": 0.6114072799682617, + "score": -0.23846659721529556 + } + }, + "time": "0:00:48.800887", + "diversity": { + "train": { + "mean_centered_corr": -0.018623020499944687, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05047355964779854 + }, + "val": { + "mean_centered_corr": -0.01870153285562992, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06864207983016968 + }, + "test": { + "mean_centered_corr": -0.019026711583137512, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09529124945402145 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/baseline_plr-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/baseline_plr-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/baseline_plr-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/baseline_plr-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..d2edb64f2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/baseline_plr-greedy-heads-evaluation/0/report.json @@ -0,0 +1,199 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22302489965783162, + "mae": 0.11715596914291382, + "r2": 0.6822243928909302, + "score": -0.22302489965783162 + }, + "val": { + "rmse": 0.23257941028161666, + "mae": 0.12573255598545074, + "r2": 0.6386785507202148, + "score": -0.23257941028161666 + }, + "test": { + "rmse": 0.23549883338311248, + "mae": 0.14942243695259094, + "r2": 0.6210193037986755, + "score": -0.23549883338311248 + } + }, + "time": "0:00:48.800887", + "diversity": { + "train": { + "mean_centered_corr": -0.018623020499944687, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05047355964779854 + }, + "val": { + "mean_centered_corr": -0.01870153285562992, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06864207983016968 + }, + "test": { + "mean_centered_corr": -0.019026711583137512, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09529124945402145 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 19, + 21, + 12, + 16, + 18, + 8, + 22, + 19, + 29 + ], + "head_selection_time": "0:00:00.158388" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/cf_fisd_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/cf_fisd_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/cf_fisd_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/cf_fisd_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..bfcad143f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/cf_fisd_only-best-head-evaluation/0/report.json @@ -0,0 +1,643 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24287177140617242, + "mae": 0.12792739272117615, + "r2": 0.6231504678726196, + "score": -0.24287177140617242 + }, + "val": { + "rmse": 0.2388197411165918, + "mae": 0.13359415531158447, + "r2": 0.61902916431427, + "score": -0.2388197411165918 + }, + "test": { + "rmse": 0.2462523116085554, + "mae": 0.16663780808448792, + "r2": 0.5856186151504517, + "score": -0.2462523116085554 + } + }, + "time": "0:01:03.015496", + "diversity": { + "train": { + "mean_centered_corr": -0.018840299919247627, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04810040444135666 + }, + "val": { + "mean_centered_corr": -0.020352300256490707, + "mean_pairwise_disagreement": NaN, + "member_std": 0.061659909784793854 + }, + "test": { + "mean_centered_corr": -0.019725581631064415, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08170976489782333 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015354" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/cf_fisd_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/cf_fisd_only-evaluation/0.toml new file mode 100644 index 000000000..523756861 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/cf_fisd_only-evaluation/0.toml @@ -0,0 +1,45 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/cf_fisd_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/cf_fisd_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/cf_fisd_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/cf_fisd_only-evaluation/0/report.json new file mode 100644 index 000000000..747fa815c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/cf_fisd_only-evaluation/0/report.json @@ -0,0 +1,639 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23279361488245573, + "mae": 0.12178773432970047, + "r2": 0.6537768840789795, + "score": -0.23279361488245573 + }, + "val": { + "rmse": 0.23523413233394516, + "mae": 0.12663741409778595, + "r2": 0.6303830146789551, + "score": -0.23523413233394516 + }, + "test": { + "rmse": 0.235149605552611, + "mae": 0.14915932714939117, + "r2": 0.6221424341201782, + "score": -0.235149605552611 + } + }, + "time": "0:01:03.015496", + "diversity": { + "train": { + "mean_centered_corr": -0.018840299919247627, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04810040444135666 + }, + "val": { + "mean_centered_corr": -0.020352300256490707, + "mean_pairwise_disagreement": NaN, + "member_std": 0.061659909784793854 + }, + "test": { + "mean_centered_corr": -0.019725581631064415, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08170976489782333 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/cf_fisd_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/cf_fisd_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/cf_fisd_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/cf_fisd_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..d85be1baa --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/cf_fisd_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,649 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23477394006764257, + "mae": 0.1248951405286789, + "r2": 0.6478613615036011, + "score": -0.23477394006764257 + }, + "val": { + "rmse": 0.23327114512588593, + "mae": 0.12512965500354767, + "r2": 0.6365260481834412, + "score": -0.23327114512588593 + }, + "test": { + "rmse": 0.23367141178994547, + "mae": 0.14663751423358917, + "r2": 0.6268780827522278, + "score": -0.23367141178994547 + } + }, + "time": "0:01:03.015496", + "diversity": { + "train": { + "mean_centered_corr": -0.018840299919247627, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04810040444135666 + }, + "val": { + "mean_centered_corr": -0.020352300256490707, + "mean_pairwise_disagreement": NaN, + "member_std": 0.061659909784793854 + }, + "test": { + "mean_centered_corr": -0.019725581631064415, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08170976489782333 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 0, + 21, + 25, + 19, + 0, + 28 + ], + "head_selection_time": "0:00:00.116743" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/esam_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/esam_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/esam_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/esam_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..0eb3e3cee --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/esam_only-best-head-evaluation/0/report.json @@ -0,0 +1,200 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/esam_only-evaluation/_esam_diagnostics/esam_only-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2276013825171354, + "mae": 0.12862664461135864, + "r2": 0.6690490245819092, + "score": -0.2276013825171354 + }, + "val": { + "rmse": 0.2398028724080995, + "mae": 0.1366567611694336, + "r2": 0.6158860921859741, + "score": -0.2398028724080995 + }, + "test": { + "rmse": 0.23711371881317633, + "mae": 0.14537189900875092, + "r2": 0.6158038973808289, + "score": -0.23711371881317633 + } + }, + "time": "0:01:29.863372", + "diversity": { + "train": { + "mean_centered_corr": -0.017507946118712425, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0488579161465168 + }, + "val": { + "mean_centered_corr": -0.017691537737846375, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0768243595957756 + }, + "test": { + "mean_centered_corr": -0.014683594927191734, + "mean_pairwise_disagreement": NaN, + "member_std": 0.10134773701429367 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19 + ], + "head_selection_time": "0:00:00.015232" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/esam_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/esam_only-evaluation/0.toml new file mode 100644 index 000000000..f5a2c1fd1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/esam_only-evaluation/0.toml @@ -0,0 +1,44 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/esam_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/esam_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/esam_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/esam_only-evaluation/0/report.json new file mode 100644 index 000000000..0401b06c5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/esam_only-evaluation/0/report.json @@ -0,0 +1,196 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/esam_only-evaluation/_esam_diagnostics/esam_only-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.21979515700408836, + "mae": 0.12170511484146118, + "r2": 0.6913614869117737, + "score": -0.21979515700408836 + }, + "val": { + "rmse": 0.23533073111501152, + "mae": 0.12932802736759186, + "r2": 0.6300793886184692, + "score": -0.23533073111501152 + }, + "test": { + "rmse": 0.24162357572341994, + "mae": 0.15881586074829102, + "r2": 0.6010502576828003, + "score": -0.24162357572341994 + } + }, + "time": "0:01:29.863372", + "diversity": { + "train": { + "mean_centered_corr": -0.017507946118712425, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0488579161465168 + }, + "val": { + "mean_centered_corr": -0.017691537737846375, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0768243595957756 + }, + "test": { + "mean_centered_corr": -0.014683594927191734, + "mean_pairwise_disagreement": NaN, + "member_std": 0.10134773701429367 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/esam_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/esam_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/esam_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/esam_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..9b882385e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/esam_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,209 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/esam_only-evaluation/_esam_diagnostics/esam_only-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.22114994595231807, + "mae": 0.12296636402606964, + "r2": 0.6875449419021606, + "score": -0.22114994595231807 + }, + "val": { + "rmse": 0.23380275644783471, + "mae": 0.12705549597740173, + "r2": 0.6348674893379211, + "score": -0.23380275644783471 + }, + "test": { + "rmse": 0.24257603366460162, + "mae": 0.1593162715435028, + "r2": 0.5978988409042358, + "score": -0.24257603366460162 + } + }, + "time": "0:01:29.863372", + "diversity": { + "train": { + "mean_centered_corr": -0.017507946118712425, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0488579161465168 + }, + "val": { + "mean_centered_corr": -0.017691537737846375, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0768243595957756 + }, + "test": { + "mean_centered_corr": -0.014683594927191734, + "mean_pairwise_disagreement": NaN, + "member_std": 0.10134773701429367 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19, + 25, + 0, + 16, + 30, + 22, + 25, + 31, + 16, + 22 + ], + "head_selection_time": "0:00:00.158598" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/mfb_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/mfb_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/mfb_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/mfb_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..9ad34d612 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/mfb_only-best-head-evaluation/0/report.json @@ -0,0 +1,226 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1184, + "metrics": { + "train": { + "rmse": 0.2105902710250961, + "mae": 0.11857529729604721, + "r2": 0.716671347618103, + "score": -0.2105902710250961 + }, + "val": { + "rmse": 0.24356476580119216, + "mae": 0.1343512386083603, + "r2": 0.6037400364875793, + "score": -0.24356476580119216 + }, + "test": { + "rmse": 0.2459548830871388, + "mae": 0.14966677129268646, + "r2": 0.5866190195083618, + "score": -0.2459548830871388 + } + }, + "time": "0:01:08.539078", + "diversity": { + "train": { + "mean_centered_corr": -0.016982607543468475, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05261818319559097 + }, + "val": { + "mean_centered_corr": -0.017025893554091454, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08680207282304764 + }, + "test": { + "mean_centered_corr": -0.01696028746664524, + "mean_pairwise_disagreement": NaN, + "member_std": 0.11552654951810837 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31 + ], + "head_selection_time": "0:00:00.015353" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/mfb_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/mfb_only-evaluation/0.toml new file mode 100644 index 000000000..da979e186 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/mfb_only-evaluation/0.toml @@ -0,0 +1,47 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/mfb_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/mfb_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/mfb_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/mfb_only-evaluation/0/report.json new file mode 100644 index 000000000..e931f6cc5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/mfb_only-evaluation/0/report.json @@ -0,0 +1,222 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1184, + "metrics": { + "train": { + "rmse": 0.20459342170445438, + "mae": 0.11115512996912003, + "r2": 0.732577919960022, + "score": -0.20459342170445438 + }, + "val": { + "rmse": 0.23520949733542967, + "mae": 0.1269780397415161, + "r2": 0.6304603815078735, + "score": -0.23520949733542967 + }, + "test": { + "rmse": 0.23880983570262945, + "mae": 0.15338461101055145, + "r2": 0.6102877855300903, + "score": -0.23880983570262945 + } + }, + "time": "0:01:08.539078", + "diversity": { + "train": { + "mean_centered_corr": -0.016982607543468475, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05261818319559097 + }, + "val": { + "mean_centered_corr": -0.017025893554091454, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08680207282304764 + }, + "test": { + "mean_centered_corr": -0.01696028746664524, + "mean_pairwise_disagreement": NaN, + "member_std": 0.11552654951810837 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/mfb_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/mfb_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/mfb_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/mfb_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..63ff10428 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/mfb_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,234 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1184, + "metrics": { + "train": { + "rmse": 0.20332438513408257, + "mae": 0.11058838665485382, + "r2": 0.7358851432800293, + "score": -0.20332438513408257 + }, + "val": { + "rmse": 0.233840053737128, + "mae": 0.12618696689605713, + "r2": 0.6347509622573853, + "score": -0.233840053737128 + }, + "test": { + "rmse": 0.23548819505307594, + "mae": 0.14876849949359894, + "r2": 0.6210535764694214, + "score": -0.23548819505307594 + } + }, + "time": "0:01:08.539078", + "diversity": { + "train": { + "mean_centered_corr": -0.016982607543468475, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05261818319559097 + }, + "val": { + "mean_centered_corr": -0.017025893554091454, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08680207282304764 + }, + "test": { + "mean_centered_corr": -0.01696028746664524, + "mean_pairwise_disagreement": NaN, + "member_std": 0.11552654951810837 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31, + 27, + 25, + 19, + 8, + 4, + 28, + 17, + 15 + ], + "head_selection_time": "0:00:00.145633" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/rla_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/rla_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/rla_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/rla_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..0877ee5ba --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/rla_only-best-head-evaluation/0/report.json @@ -0,0 +1,203 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2357054301684337, + "mae": 0.12653841078281403, + "r2": 0.6450614929199219, + "score": -0.2357054301684337 + }, + "val": { + "rmse": 0.23765212789668771, + "mae": 0.13472485542297363, + "r2": 0.6227452754974365, + "score": -0.23765212789668771 + }, + "test": { + "rmse": 0.24778267915496135, + "mae": 0.16990186274051666, + "r2": 0.5804521441459656, + "score": -0.24778267915496135 + } + }, + "time": "0:01:33.893743", + "diversity": { + "train": { + "mean_centered_corr": -0.018528519198298454, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045790109783411026 + }, + "val": { + "mean_centered_corr": -0.01869109645485878, + "mean_pairwise_disagreement": NaN, + "member_std": 0.057747576385736465 + }, + "test": { + "mean_centered_corr": -0.017511537298560143, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07643643766641617 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 18 + ], + "head_selection_time": "0:00:00.015586" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/rla_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/rla_only-evaluation/0.toml new file mode 100644 index 000000000..485d94624 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/rla_only-evaluation/0.toml @@ -0,0 +1,41 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/rla_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/rla_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/rla_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/rla_only-evaluation/0/report.json new file mode 100644 index 000000000..644da59b3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/rla_only-evaluation/0/report.json @@ -0,0 +1,199 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2313646747453307, + "mae": 0.12118230015039444, + "r2": 0.6580142974853516, + "score": -0.2313646747453307 + }, + "val": { + "rmse": 0.2354418316334552, + "mae": 0.1258188933134079, + "r2": 0.6297299861907959, + "score": -0.2354418316334552 + }, + "test": { + "rmse": 0.23162903716515967, + "mae": 0.14173103868961334, + "r2": 0.6333720684051514, + "score": -0.23162903716515967 + } + }, + "time": "0:01:33.893743", + "diversity": { + "train": { + "mean_centered_corr": -0.018528519198298454, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045790109783411026 + }, + "val": { + "mean_centered_corr": -0.01869109645485878, + "mean_pairwise_disagreement": NaN, + "member_std": 0.057747576385736465 + }, + "test": { + "mean_centered_corr": -0.017511537298560143, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07643643766641617 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/rla_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/rla_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/rla_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/rla_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..d9d2a13ce --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/rla_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,208 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23082056510008958, + "mae": 0.12217259407043457, + "r2": 0.6596208810806274, + "score": -0.23082056510008958 + }, + "val": { + "rmse": 0.2336025622310011, + "mae": 0.12522470951080322, + "r2": 0.6354925036430359, + "score": -0.2336025622310011 + }, + "test": { + "rmse": 0.23389241285301454, + "mae": 0.14725922048091888, + "r2": 0.6261719465255737, + "score": -0.23389241285301454 + } + }, + "time": "0:01:33.893743", + "diversity": { + "train": { + "mean_centered_corr": -0.018528519198298454, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045790109783411026 + }, + "val": { + "mean_centered_corr": -0.01869109645485878, + "mean_pairwise_disagreement": NaN, + "member_std": 0.057747576385736465 + }, + "test": { + "mean_centered_corr": -0.017511537298560143, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07643643766641617 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 18, + 21, + 19, + 25, + 6, + 4 + ], + "head_selection_time": "0:00:00.102958" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_four_combined-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_four_combined-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_four_combined-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_four_combined-best-head-evaluation/0/report.json new file mode 100644 index 000000000..de85e5c86 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_four_combined-best-head-evaluation/0/report.json @@ -0,0 +1,455 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9000651240348816, + "min_features_per_member": 164.0, + "mean_features_per_member": 172.8125, + "max_features_per_member": 181.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.80208396911621, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8173701129083908, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_four_combined-evaluation/_esam_diagnostics/all_four_combined-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.4515053148590503, + "mae": 0.35153692960739136, + "r2": 0.5317407250404358, + "score": -0.4515053148590503 + }, + "val": { + "rmse": 0.46391938568359065, + "mae": 0.36074763536453247, + "r2": 0.5064277052879333, + "score": -0.46391938568359065 + }, + "test": { + "rmse": 0.48142889277254225, + "mae": 0.3758453130722046, + "r2": 0.4832448363304138, + "score": -0.48142889277254225 + } + }, + "time": "0:38:21.569603", + "diversity": { + "train": { + "mean_centered_corr": -0.009637685492634773, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04498574882745743 + }, + "val": { + "mean_centered_corr": -0.009298564866185188, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046711746603250504 + }, + "test": { + "mean_centered_corr": -0.00908930879086256, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04780808091163635 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 25 + ], + "head_selection_time": "0:00:00.028483" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_four_combined-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_four_combined-evaluation/0.toml new file mode 100644 index 000000000..7de7ee9ce --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_four_combined-evaluation/0.toml @@ -0,0 +1,74 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 8 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "cooking-time" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_four_combined-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_four_combined-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_four_combined-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_four_combined-evaluation/0/report.json new file mode 100644 index 000000000..e3ee653ca --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_four_combined-evaluation/0/report.json @@ -0,0 +1,451 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9000651240348816, + "min_features_per_member": 164.0, + "mean_features_per_member": 172.8125, + "max_features_per_member": 181.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.80208396911621, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8173701129083908, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_four_combined-evaluation/_esam_diagnostics/all_four_combined-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.44819067366049176, + "mae": 0.3488732874393463, + "r2": 0.5385907888412476, + "score": -0.44819067366049176 + }, + "val": { + "rmse": 0.463355012524859, + "mae": 0.3601790964603424, + "r2": 0.5076278448104858, + "score": -0.463355012524859 + }, + "test": { + "rmse": 0.4809202538385541, + "mae": 0.3753686547279358, + "r2": 0.48433613777160645, + "score": -0.4809202538385541 + } + }, + "time": "0:38:21.569603", + "diversity": { + "train": { + "mean_centered_corr": -0.009637685492634773, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04498574882745743 + }, + "val": { + "mean_centered_corr": -0.009298564866185188, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046711746603250504 + }, + "test": { + "mean_centered_corr": -0.00908930879086256, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04780808091163635 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_four_combined-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_four_combined-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_four_combined-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_four_combined-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..19d838edb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_four_combined-greedy-heads-evaluation/0/report.json @@ -0,0 +1,464 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9000651240348816, + "min_features_per_member": 164.0, + "mean_features_per_member": 172.8125, + "max_features_per_member": 181.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.80208396911621, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8173701129083908, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_four_combined-evaluation/_esam_diagnostics/all_four_combined-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.44759809638897813, + "mae": 0.3485293388366699, + "r2": 0.539810061454773, + "score": -0.44759809638897813 + }, + "val": { + "rmse": 0.4629352081332738, + "mae": 0.35992681980133057, + "r2": 0.5085196495056152, + "score": -0.4629352081332738 + }, + "test": { + "rmse": 0.4802514336856013, + "mae": 0.3748669922351837, + "r2": 0.48576945066452026, + "score": -0.4802514336856013 + } + }, + "time": "0:38:21.569603", + "diversity": { + "train": { + "mean_centered_corr": -0.009637685492634773, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04498574882745743 + }, + "val": { + "mean_centered_corr": -0.009298564866185188, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046711746603250504 + }, + "test": { + "mean_centered_corr": -0.00908930879086256, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04780808091163635 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 25, + 24, + 19, + 1, + 21, + 27, + 15, + 2, + 12, + 31 + ], + "head_selection_time": "0:00:00.370342" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_cf_fisd-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_cf_fisd-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_cf_fisd-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_cf_fisd-best-head-evaluation/0/report.json new file mode 100644 index 000000000..82f9fc957 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_cf_fisd-best-head-evaluation/0/report.json @@ -0,0 +1,202 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9000651240348816, + "min_features_per_member": 164.0, + "mean_features_per_member": 172.8125, + "max_features_per_member": 181.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.80208396911621, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8173701129083908, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_cf_fisd-evaluation/_esam_diagnostics/all_minus_cf_fisd-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.44844456178640835, + "mae": 0.34928542375564575, + "r2": 0.5380678772926331, + "score": -0.44844456178640835 + }, + "val": { + "rmse": 0.4649621508176366, + "mae": 0.3616116940975189, + "r2": 0.5042064189910889, + "score": -0.4649621508176366 + }, + "test": { + "rmse": 0.48187581714110606, + "mae": 0.3764643371105194, + "r2": 0.48228490352630615, + "score": -0.48187581714110606 + } + }, + "time": "0:33:40.844452", + "diversity": { + "train": { + "mean_centered_corr": -0.020041001960635185, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04958989471197128 + }, + "val": { + "mean_centered_corr": -0.019671577960252762, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05151689797639847 + }, + "test": { + "mean_centered_corr": -0.019756164401769638, + "mean_pairwise_disagreement": NaN, + "member_std": 0.052489347755908966 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 18 + ], + "head_selection_time": "0:00:00.028038" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_cf_fisd-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_cf_fisd-evaluation/0.toml new file mode 100644 index 000000000..ccda6b804 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_cf_fisd-evaluation/0.toml @@ -0,0 +1,63 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_cf_fisd-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_cf_fisd-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_cf_fisd-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_cf_fisd-evaluation/0/report.json new file mode 100644 index 000000000..dac75d544 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_cf_fisd-evaluation/0/report.json @@ -0,0 +1,198 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9000651240348816, + "min_features_per_member": 164.0, + "mean_features_per_member": 172.8125, + "max_features_per_member": 181.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.80208396911621, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8173701129083908, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_cf_fisd-evaluation/_esam_diagnostics/all_minus_cf_fisd-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.44612866614094726, + "mae": 0.3472161889076233, + "r2": 0.5428266525268555, + "score": -0.44612866614094726 + }, + "val": { + "rmse": 0.463067499804912, + "mae": 0.35988649725914, + "r2": 0.5082387328147888, + "score": -0.463067499804912 + }, + "test": { + "rmse": 0.4804862724768644, + "mae": 0.3750061094760895, + "r2": 0.48526638746261597, + "score": -0.4804862724768644 + } + }, + "time": "0:33:40.844452", + "diversity": { + "train": { + "mean_centered_corr": -0.020041001960635185, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04958989471197128 + }, + "val": { + "mean_centered_corr": -0.019671577960252762, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05151689797639847 + }, + "test": { + "mean_centered_corr": -0.019756164401769638, + "mean_pairwise_disagreement": NaN, + "member_std": 0.052489347755908966 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_cf_fisd-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_cf_fisd-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_cf_fisd-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_cf_fisd-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..a026853c9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_cf_fisd-greedy-heads-evaluation/0/report.json @@ -0,0 +1,219 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9000651240348816, + "min_features_per_member": 164.0, + "mean_features_per_member": 172.8125, + "max_features_per_member": 181.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.80208396911621, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8173701129083908, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_cf_fisd-evaluation/_esam_diagnostics/all_minus_cf_fisd-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.4463454196329557, + "mae": 0.3473359942436218, + "r2": 0.5423822999000549, + "score": -0.4463454196329557 + }, + "val": { + "rmse": 0.46276401446104365, + "mae": 0.3595953583717346, + "r2": 0.5088831186294556, + "score": -0.46276401446104365 + }, + "test": { + "rmse": 0.4800152693247022, + "mae": 0.37460875511169434, + "r2": 0.4862750768661499, + "score": -0.4800152693247022 + } + }, + "time": "0:33:40.844452", + "diversity": { + "train": { + "mean_centered_corr": -0.020041001960635185, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04958989471197128 + }, + "val": { + "mean_centered_corr": -0.019671577960252762, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05151689797639847 + }, + "test": { + "mean_centered_corr": -0.019756164401769638, + "mean_pairwise_disagreement": NaN, + "member_std": 0.052489347755908966 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 18, + 19, + 25, + 14, + 26, + 11, + 3, + 27, + 16, + 5, + 12, + 21, + 29, + 14, + 26, + 24, + 25, + 19 + ], + "head_selection_time": "0:00:00.765242" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_esam-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_esam-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_esam-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_esam-best-head-evaluation/0/report.json new file mode 100644 index 000000000..82580a9ac --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_esam-best-head-evaluation/0/report.json @@ -0,0 +1,445 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9000651240348816, + "min_features_per_member": 164.0, + "mean_features_per_member": 172.8125, + "max_features_per_member": 181.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.80208396911621, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8173701129083908, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.45139719963574126, + "mae": 0.35117724537849426, + "r2": 0.5319649577140808, + "score": -0.45139719963574126 + }, + "val": { + "rmse": 0.46397896473985684, + "mae": 0.36049172282218933, + "r2": 0.5063009262084961, + "score": -0.46397896473985684 + }, + "test": { + "rmse": 0.481064806640852, + "mae": 0.3752272427082062, + "r2": 0.48402613401412964, + "score": -0.481064806640852 + } + }, + "time": "0:23:29.845146", + "diversity": { + "train": { + "mean_centered_corr": -0.009395277127623558, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045357972383499146 + }, + "val": { + "mean_centered_corr": -0.008926574140787125, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04711531847715378 + }, + "test": { + "mean_centered_corr": -0.009024984203279018, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04819539934396744 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 24 + ], + "head_selection_time": "0:00:00.027833" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_esam-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_esam-evaluation/0.toml new file mode 100644 index 000000000..fbe196209 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_esam-evaluation/0.toml @@ -0,0 +1,64 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 8 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "cooking-time" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_esam-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_esam-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_esam-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_esam-evaluation/0/report.json new file mode 100644 index 000000000..30473de49 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_esam-evaluation/0/report.json @@ -0,0 +1,441 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9000651240348816, + "min_features_per_member": 164.0, + "mean_features_per_member": 172.8125, + "max_features_per_member": 181.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.80208396911621, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8173701129083908, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.44808515070926874, + "mae": 0.3487781286239624, + "r2": 0.5388079881668091, + "score": -0.44808515070926874 + }, + "val": { + "rmse": 0.4633378873945458, + "mae": 0.3601687252521515, + "r2": 0.5076642632484436, + "score": -0.4633378873945458 + }, + "test": { + "rmse": 0.48088029743109706, + "mae": 0.37534818053245544, + "r2": 0.48442184925079346, + "score": -0.48088029743109706 + } + }, + "time": "0:23:29.845146", + "diversity": { + "train": { + "mean_centered_corr": -0.009395277127623558, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045357972383499146 + }, + "val": { + "mean_centered_corr": -0.008926574140787125, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04711531847715378 + }, + "test": { + "mean_centered_corr": -0.009024984203279018, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04819539934396744 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_esam-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_esam-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_esam-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_esam-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..8f37ba891 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_esam-greedy-heads-evaluation/0/report.json @@ -0,0 +1,457 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9000651240348816, + "min_features_per_member": 164.0, + "mean_features_per_member": 172.8125, + "max_features_per_member": 181.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.80208396911621, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8173701129083908, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.4475514693736722, + "mae": 0.3484344184398651, + "r2": 0.5399059653282166, + "score": -0.4475514693736722 + }, + "val": { + "rmse": 0.4629167637909319, + "mae": 0.3598645031452179, + "r2": 0.5085588097572327, + "score": -0.4629167637909319 + }, + "test": { + "rmse": 0.4803221564231222, + "mae": 0.3749236762523651, + "r2": 0.485617995262146, + "score": -0.4803221564231222 + } + }, + "time": "0:23:29.845146", + "diversity": { + "train": { + "mean_centered_corr": -0.009395277127623558, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045357972383499146 + }, + "val": { + "mean_centered_corr": -0.008926574140787125, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04711531847715378 + }, + "test": { + "mean_centered_corr": -0.009024984203279018, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04819539934396744 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 24, + 27, + 19, + 2, + 12, + 25, + 1, + 11, + 31, + 14, + 27, + 2, + 15 + ], + "head_selection_time": "0:00:00.488491" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_mfb-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_mfb-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_mfb-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_mfb-best-head-evaluation/0/report.json new file mode 100644 index 000000000..de00f541b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_mfb-best-head-evaluation/0/report.json @@ -0,0 +1,419 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_mfb-evaluation/_esam_diagnostics/all_minus_mfb-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.451333797182134, + "mae": 0.3511902987957001, + "r2": 0.5320963859558105, + "score": -0.451333797182134 + }, + "val": { + "rmse": 0.46380663024741176, + "mae": 0.36054760217666626, + "r2": 0.5066676139831543, + "score": -0.46380663024741176 + }, + "test": { + "rmse": 0.48139068107848576, + "mae": 0.37560245394706726, + "r2": 0.48332685232162476, + "score": -0.48139068107848576 + } + }, + "time": "0:35:13.729663", + "diversity": { + "train": { + "mean_centered_corr": -0.005216025747358799, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04148838296532631 + }, + "val": { + "mean_centered_corr": -0.005656818859279156, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0424356609582901 + }, + "test": { + "mean_centered_corr": -0.005289691034704447, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0431172214448452 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 24 + ], + "head_selection_time": "0:00:00.028966" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_mfb-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_mfb-evaluation/0.toml new file mode 100644 index 000000000..f4e6abf99 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_mfb-evaluation/0.toml @@ -0,0 +1,61 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "cooking-time" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_mfb-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_mfb-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_mfb-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_mfb-evaluation/0/report.json new file mode 100644 index 000000000..e9f3634d6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_mfb-evaluation/0/report.json @@ -0,0 +1,415 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_mfb-evaluation/_esam_diagnostics/all_minus_mfb-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.44820339059913267, + "mae": 0.3488495647907257, + "r2": 0.5385645627975464, + "score": -0.44820339059913267 + }, + "val": { + "rmse": 0.46319753395446966, + "mae": 0.3600589334964752, + "r2": 0.5079624652862549, + "score": -0.46319753395446966 + }, + "test": { + "rmse": 0.48088071575887986, + "mae": 0.37532681226730347, + "r2": 0.4844209551811218, + "score": -0.48088071575887986 + } + }, + "time": "0:35:13.729663", + "diversity": { + "train": { + "mean_centered_corr": -0.005216025747358799, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04148838296532631 + }, + "val": { + "mean_centered_corr": -0.005656818859279156, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0424356609582901 + }, + "test": { + "mean_centered_corr": -0.005289691034704447, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0431172214448452 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_mfb-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_mfb-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_mfb-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_mfb-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..e82fce40f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_mfb-greedy-heads-evaluation/0/report.json @@ -0,0 +1,434 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_mfb-evaluation/_esam_diagnostics/all_minus_mfb-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.4478290457374166, + "mae": 0.348653107881546, + "r2": 0.5393350124359131, + "score": -0.4478290457374166 + }, + "val": { + "rmse": 0.4628484521639483, + "mae": 0.3599395155906677, + "r2": 0.5087038278579712, + "score": -0.4628484521639483 + }, + "test": { + "rmse": 0.4805722784138239, + "mae": 0.37523025274276733, + "r2": 0.48508208990097046, + "score": -0.4805722784138239 + } + }, + "time": "0:35:13.729663", + "diversity": { + "train": { + "mean_centered_corr": -0.005216025747358799, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04148838296532631 + }, + "val": { + "mean_centered_corr": -0.005656818859279156, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0424356609582901 + }, + "test": { + "mean_centered_corr": -0.005289691034704447, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0431172214448452 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 24, + 14, + 2, + 31, + 22, + 21, + 25, + 14, + 2, + 19, + 27, + 31, + 12, + 2, + 14, + 3 + ], + "head_selection_time": "0:00:00.651502" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_rla-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_rla-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_rla-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_rla-best-head-evaluation/0/report.json new file mode 100644 index 000000000..9055fe557 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_rla-best-head-evaluation/0/report.json @@ -0,0 +1,446 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9000651240348816, + "min_features_per_member": 164.0, + "mean_features_per_member": 172.8125, + "max_features_per_member": 181.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.80208396911621, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8173701129083908, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_rla-evaluation/_esam_diagnostics/all_minus_rla-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.45204026182588536, + "mae": 0.3519219160079956, + "r2": 0.5306304693222046, + "score": -0.45204026182588536 + }, + "val": { + "rmse": 0.4640838759408798, + "mae": 0.36102116107940674, + "r2": 0.5060776472091675, + "score": -0.4640838759408798 + }, + "test": { + "rmse": 0.48156976542107044, + "mae": 0.37596550583839417, + "r2": 0.4829423427581787, + "score": -0.48156976542107044 + } + }, + "time": "0:20:21.935298", + "diversity": { + "train": { + "mean_centered_corr": -0.009009289555251598, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04566791653633118 + }, + "val": { + "mean_centered_corr": -0.008741805329918861, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04742306470870972 + }, + "test": { + "mean_centered_corr": -0.008637306280434132, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04842785745859146 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 25 + ], + "head_selection_time": "0:00:00.029409" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_rla-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_rla-evaluation/0.toml new file mode 100644 index 000000000..cdf7f3e0f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_rla-evaluation/0.toml @@ -0,0 +1,67 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 8 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "cooking-time" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_rla-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_rla-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_rla-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_rla-evaluation/0/report.json new file mode 100644 index 000000000..4cd78cf83 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_rla-evaluation/0/report.json @@ -0,0 +1,442 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9000651240348816, + "min_features_per_member": 164.0, + "mean_features_per_member": 172.8125, + "max_features_per_member": 181.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.80208396911621, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8173701129083908, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_rla-evaluation/_esam_diagnostics/all_minus_rla-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.4485826549798914, + "mae": 0.34918203949928284, + "r2": 0.5377833247184753, + "score": -0.4485826549798914 + }, + "val": { + "rmse": 0.4634391174751191, + "mae": 0.3602841794490814, + "r2": 0.5074490904808044, + "score": -0.4634391174751191 + }, + "test": { + "rmse": 0.4810216869909678, + "mae": 0.3754705488681793, + "r2": 0.48411864042282104, + "score": -0.4810216869909678 + } + }, + "time": "0:20:21.935298", + "diversity": { + "train": { + "mean_centered_corr": -0.009009289555251598, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04566791653633118 + }, + "val": { + "mean_centered_corr": -0.008741805329918861, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04742306470870972 + }, + "test": { + "mean_centered_corr": -0.008637306280434132, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04842785745859146 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_rla-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_rla-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_rla-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_rla-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..a875673e6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_rla-greedy-heads-evaluation/0/report.json @@ -0,0 +1,461 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9000651240348816, + "min_features_per_member": 164.0, + "mean_features_per_member": 172.8125, + "max_features_per_member": 181.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.80208396911621, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8173701129083908, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_rla-evaluation/_esam_diagnostics/all_minus_rla-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.44756392147614626, + "mae": 0.3484703004360199, + "r2": 0.5398803949356079, + "score": -0.44756392147614626 + }, + "val": { + "rmse": 0.46292271885090186, + "mae": 0.3598572313785553, + "r2": 0.5085461735725403, + "score": -0.46292271885090186 + }, + "test": { + "rmse": 0.48053390557368486, + "mae": 0.3750901222229004, + "r2": 0.4851643443107605, + "score": -0.48053390557368486 + } + }, + "time": "0:20:21.935298", + "diversity": { + "train": { + "mean_centered_corr": -0.009009289555251598, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04566791653633118 + }, + "val": { + "mean_centered_corr": -0.008741805329918861, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04742306470870972 + }, + "test": { + "mean_centered_corr": -0.008637306280434132, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04842785745859146 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 25, + 27, + 15, + 2, + 13, + 24, + 21, + 31, + 19, + 1, + 11, + 12, + 27, + 15, + 2, + 24 + ], + "head_selection_time": "0:00:00.649511" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.05-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.05-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.05-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.05-best-head-evaluation/0/report.json new file mode 100644 index 000000000..222652a2b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.05-best-head-evaluation/0/report.json @@ -0,0 +1,400 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.45199979641686133, + "mae": 0.3517935872077942, + "r2": 0.5307145118713379, + "score": -0.45199979641686133 + }, + "val": { + "rmse": 0.4638214410008231, + "mae": 0.360493928194046, + "r2": 0.5066360831260681, + "score": -0.4638214410008231 + }, + "test": { + "rmse": 0.4819659035779245, + "mae": 0.37611493468284607, + "r2": 0.48209136724472046, + "score": -0.4819659035779245 + } + }, + "time": "0:10:10.466567", + "diversity": { + "train": { + "mean_centered_corr": -0.005496397614479065, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04279099032282829 + }, + "val": { + "mean_centered_corr": -0.005842795129865408, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04372712969779968 + }, + "test": { + "mean_centered_corr": -0.005908838007599115, + "mean_pairwise_disagreement": NaN, + "member_std": 0.044375959783792496 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31 + ], + "head_selection_time": "0:00:00.027383" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.05-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.05-evaluation/0.toml new file mode 100644 index 000000000..784c17199 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.05-evaluation/0.toml @@ -0,0 +1,44 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 + +[cf_fisd] +lambda = 0.05 +variant = "raw" +dataset_name = "cooking-time" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.05-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.05-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.05-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.05-evaluation/0/report.json new file mode 100644 index 000000000..f58c2dd92 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.05-evaluation/0/report.json @@ -0,0 +1,396 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.4482607702587857, + "mae": 0.34888917207717896, + "r2": 0.5384464263916016, + "score": -0.4482607702587857 + }, + "val": { + "rmse": 0.4632019090821483, + "mae": 0.3600703775882721, + "r2": 0.5079531669616699, + "score": -0.4632019090821483 + }, + "test": { + "rmse": 0.4808496810468957, + "mae": 0.3752937614917755, + "r2": 0.48448747396469116, + "score": -0.4808496810468957 + } + }, + "time": "0:10:10.466567", + "diversity": { + "train": { + "mean_centered_corr": -0.005496397614479065, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04279099032282829 + }, + "val": { + "mean_centered_corr": -0.005842795129865408, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04372712969779968 + }, + "test": { + "mean_centered_corr": -0.005908838007599115, + "mean_pairwise_disagreement": NaN, + "member_std": 0.044375959783792496 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.05-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.05-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.05-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.05-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..0f7f787a9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.05-greedy-heads-evaluation/0/report.json @@ -0,0 +1,413 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.44782199154768965, + "mae": 0.34862202405929565, + "r2": 0.5393495559692383, + "score": -0.44782199154768965 + }, + "val": { + "rmse": 0.4628482268025946, + "mae": 0.35992974042892456, + "r2": 0.5087043046951294, + "score": -0.4628482268025946 + }, + "test": { + "rmse": 0.4803119031248436, + "mae": 0.3749502897262573, + "r2": 0.48563992977142334, + "score": -0.4803119031248436 + } + }, + "time": "0:10:10.466567", + "diversity": { + "train": { + "mean_centered_corr": -0.005496397614479065, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04279099032282829 + }, + "val": { + "mean_centered_corr": -0.005842795129865408, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04372712969779968 + }, + "test": { + "mean_centered_corr": -0.005908838007599115, + "mean_pairwise_disagreement": NaN, + "member_std": 0.044375959783792496 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31, + 27, + 14, + 2, + 21, + 25, + 2, + 12, + 22, + 14, + 27, + 2, + 19, + 24 + ], + "head_selection_time": "0:00:00.542336" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.1-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.1-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.1-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.1-best-head-evaluation/0/report.json new file mode 100644 index 000000000..8007cdd2c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.1-best-head-evaluation/0/report.json @@ -0,0 +1,400 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.45293358344227996, + "mae": 0.35263803601264954, + "r2": 0.5287735462188721, + "score": -0.45293358344227996 + }, + "val": { + "rmse": 0.46397138529545745, + "mae": 0.3607122600078583, + "r2": 0.5063170194625854, + "score": -0.46397138529545745 + }, + "test": { + "rmse": 0.4813020969034895, + "mae": 0.3759235143661499, + "r2": 0.48351699113845825, + "score": -0.4813020969034895 + } + }, + "time": "0:10:10.580086", + "diversity": { + "train": { + "mean_centered_corr": -0.004884317051619291, + "mean_pairwise_disagreement": NaN, + "member_std": 0.042778123170137405 + }, + "val": { + "mean_centered_corr": -0.005044749937951565, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04371502622961998 + }, + "test": { + "mean_centered_corr": -0.005197949707508087, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04438793286681175 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 2 + ], + "head_selection_time": "0:00:00.027731" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.1-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.1-evaluation/0.toml new file mode 100644 index 000000000..a22753f01 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.1-evaluation/0.toml @@ -0,0 +1,44 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "cooking-time" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.1-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.1-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.1-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.1-evaluation/0/report.json new file mode 100644 index 000000000..5f2deca4b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.1-evaluation/0/report.json @@ -0,0 +1,396 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.4484508087097321, + "mae": 0.34904322028160095, + "r2": 0.5380549430847168, + "score": -0.4484508087097321 + }, + "val": { + "rmse": 0.4632809116304394, + "mae": 0.36012470722198486, + "r2": 0.5077853202819824, + "score": -0.4632809116304394 + }, + "test": { + "rmse": 0.48091704691326154, + "mae": 0.3753390908241272, + "r2": 0.4843430519104004, + "score": -0.48091704691326154 + } + }, + "time": "0:10:10.580086", + "diversity": { + "train": { + "mean_centered_corr": -0.004884317051619291, + "mean_pairwise_disagreement": NaN, + "member_std": 0.042778123170137405 + }, + "val": { + "mean_centered_corr": -0.005044749937951565, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04371502622961998 + }, + "test": { + "mean_centered_corr": -0.005197949707508087, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04438793286681175 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.1-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.1-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.1-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.1-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..7b442d96c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.1-greedy-heads-evaluation/0/report.json @@ -0,0 +1,403 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.44943641696524117, + "mae": 0.34985706210136414, + "r2": 0.5360222458839417, + "score": -0.44943641696524117 + }, + "val": { + "rmse": 0.4629530240908241, + "mae": 0.35998642444610596, + "r2": 0.5084818601608276, + "score": -0.4629530240908241 + }, + "test": { + "rmse": 0.480628211999089, + "mae": 0.37514716386795044, + "r2": 0.48496222496032715, + "score": -0.480628211999089 + } + }, + "time": "0:10:10.580086", + "diversity": { + "train": { + "mean_centered_corr": -0.004884317051619291, + "mean_pairwise_disagreement": NaN, + "member_std": 0.042778123170137405 + }, + "val": { + "mean_centered_corr": -0.005044749937951565, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04371502622961998 + }, + "test": { + "mean_centered_corr": -0.005197949707508087, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04438793286681175 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 2, + 14, + 24, + 31 + ], + "head_selection_time": "0:00:00.141449" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.2-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.2-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.2-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.2-best-head-evaluation/0/report.json new file mode 100644 index 000000000..3b365e9ce --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.2-best-head-evaluation/0/report.json @@ -0,0 +1,400 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.2, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.2, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 15096, + "metrics": { + "train": { + "rmse": 0.4517256240823104, + "mae": 0.35139286518096924, + "r2": 0.5312836170196533, + "score": -0.4517256240823104 + }, + "val": { + "rmse": 0.4641009735445309, + "mae": 0.36067935824394226, + "r2": 0.5060412287712097, + "score": -0.4641009735445309 + }, + "test": { + "rmse": 0.48193517064355434, + "mae": 0.3760278820991516, + "r2": 0.4821573495864868, + "score": -0.48193517064355434 + } + }, + "time": "0:08:57.117859", + "diversity": { + "train": { + "mean_centered_corr": -0.01122801098972559, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04195729270577431 + }, + "val": { + "mean_centered_corr": -0.010996237397193909, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04290297254920006 + }, + "test": { + "mean_centered_corr": -0.011064938269555569, + "mean_pairwise_disagreement": NaN, + "member_std": 0.043424952775239944 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 24 + ], + "head_selection_time": "0:00:00.028283" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.2-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.2-evaluation/0.toml new file mode 100644 index 000000000..5cc98e607 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.2-evaluation/0.toml @@ -0,0 +1,44 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 + +[cf_fisd] +lambda = 0.2 +variant = "raw" +dataset_name = "cooking-time" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.2-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.2-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.2-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.2-evaluation/0/report.json new file mode 100644 index 000000000..6074e8c77 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.2-evaluation/0/report.json @@ -0,0 +1,396 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.2, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.2, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 15096, + "metrics": { + "train": { + "rmse": 0.44928435800197336, + "mae": 0.3496941924095154, + "r2": 0.5363361239433289, + "score": -0.44928435800197336 + }, + "val": { + "rmse": 0.4633341406854049, + "mae": 0.3601973056793213, + "r2": 0.5076721906661987, + "score": -0.4633341406854049 + }, + "test": { + "rmse": 0.4808806847716492, + "mae": 0.3753437399864197, + "r2": 0.4844210147857666, + "score": -0.4808806847716492 + } + }, + "time": "0:08:57.117859", + "diversity": { + "train": { + "mean_centered_corr": -0.01122801098972559, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04195729270577431 + }, + "val": { + "mean_centered_corr": -0.010996237397193909, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04290297254920006 + }, + "test": { + "mean_centered_corr": -0.011064938269555569, + "mean_pairwise_disagreement": NaN, + "member_std": 0.043424952775239944 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.2-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.2-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.2-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.2-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..b663642e5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.2-greedy-heads-evaluation/0/report.json @@ -0,0 +1,409 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.2, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.2, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 15096, + "metrics": { + "train": { + "rmse": 0.44893214163383177, + "mae": 0.3495011031627655, + "r2": 0.5370628237724304, + "score": -0.44893214163383177 + }, + "val": { + "rmse": 0.463044314080512, + "mae": 0.360007643699646, + "r2": 0.5082879662513733, + "score": -0.463044314080512 + }, + "test": { + "rmse": 0.4804926610439783, + "mae": 0.37509334087371826, + "r2": 0.4852526783943176, + "score": -0.4804926610439783 + } + }, + "time": "0:08:57.117859", + "diversity": { + "train": { + "mean_centered_corr": -0.01122801098972559, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04195729270577431 + }, + "val": { + "mean_centered_corr": -0.010996237397193909, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04290297254920006 + }, + "test": { + "mean_centered_corr": -0.011064938269555569, + "mean_pairwise_disagreement": NaN, + "member_std": 0.043424952775239944 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 24, + 6, + 17, + 22, + 12, + 6, + 21, + 27, + 13, + 6 + ], + "head_selection_time": "0:00:00.354807" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_cf_fisd-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_cf_fisd-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_cf_fisd-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_cf_fisd-best-head-evaluation/0/report.json new file mode 100644 index 000000000..72d2eed76 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_cf_fisd-best-head-evaluation/0/report.json @@ -0,0 +1,410 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_cf_fisd-evaluation/_esam_diagnostics/esam_cf_fisd-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.45234552294131686, + "mae": 0.35200172662734985, + "r2": 0.5299963355064392, + "score": -0.45234552294131686 + }, + "val": { + "rmse": 0.4639245891190366, + "mae": 0.3605712652206421, + "r2": 0.5064166784286499, + "score": -0.4639245891190366 + }, + "test": { + "rmse": 0.48198567488759675, + "mae": 0.3760890066623688, + "r2": 0.48204880952835083, + "score": -0.48198567488759675 + } + }, + "time": "0:17:32.517766", + "diversity": { + "train": { + "mean_centered_corr": -0.004823723807930946, + "mean_pairwise_disagreement": NaN, + "member_std": 0.042313702404499054 + }, + "val": { + "mean_centered_corr": -0.005078355316072702, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04322447255253792 + }, + "test": { + "mean_centered_corr": -0.005122650880366564, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04387650638818741 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31 + ], + "head_selection_time": "0:00:00.028152" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_cf_fisd-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_cf_fisd-evaluation/0.toml new file mode 100644 index 000000000..53fd18ac6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_cf_fisd-evaluation/0.toml @@ -0,0 +1,54 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "cooking-time" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_cf_fisd-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_cf_fisd-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_cf_fisd-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_cf_fisd-evaluation/0/report.json new file mode 100644 index 000000000..c93a16e5b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_cf_fisd-evaluation/0/report.json @@ -0,0 +1,406 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_cf_fisd-evaluation/_esam_diagnostics/esam_cf_fisd-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.44859881540040863, + "mae": 0.34915608167648315, + "r2": 0.5377500057220459, + "score": -0.44859881540040863 + }, + "val": { + "rmse": 0.4632865725335359, + "mae": 0.3601624369621277, + "r2": 0.5077732801437378, + "score": -0.4632865725335359 + }, + "test": { + "rmse": 0.48093585437297304, + "mae": 0.37536507844924927, + "r2": 0.48430269956588745, + "score": -0.48093585437297304 + } + }, + "time": "0:17:32.517766", + "diversity": { + "train": { + "mean_centered_corr": -0.004823723807930946, + "mean_pairwise_disagreement": NaN, + "member_std": 0.042313702404499054 + }, + "val": { + "mean_centered_corr": -0.005078355316072702, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04322447255253792 + }, + "test": { + "mean_centered_corr": -0.005122650880366564, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04387650638818741 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_cf_fisd-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_cf_fisd-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_cf_fisd-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_cf_fisd-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..21155c67f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_cf_fisd-greedy-heads-evaluation/0/report.json @@ -0,0 +1,413 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_cf_fisd-evaluation/_esam_diagnostics/esam_cf_fisd-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.44944187096507476, + "mae": 0.34989649057388306, + "r2": 0.5360109806060791, + "score": -0.44944187096507476 + }, + "val": { + "rmse": 0.4629679265241866, + "mae": 0.3600855767726898, + "r2": 0.5084501504898071, + "score": -0.4629679265241866 + }, + "test": { + "rmse": 0.4805148498004945, + "mae": 0.37517452239990234, + "r2": 0.48520517349243164, + "score": -0.4805148498004945 + } + }, + "time": "0:17:32.517766", + "diversity": { + "train": { + "mean_centered_corr": -0.004823723807930946, + "mean_pairwise_disagreement": NaN, + "member_std": 0.042313702404499054 + }, + "val": { + "mean_centered_corr": -0.005078355316072702, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04322447255253792 + }, + "test": { + "mean_centered_corr": -0.005122650880366564, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04387650638818741 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31, + 14, + 2, + 22 + ], + "head_selection_time": "0:00:00.142880" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_mfb-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_mfb-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_mfb-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_mfb-best-head-evaluation/0/report.json new file mode 100644 index 000000000..7207b878b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_mfb-best-head-evaluation/0/report.json @@ -0,0 +1,193 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9000651240348816, + "min_features_per_member": 164.0, + "mean_features_per_member": 172.8125, + "max_features_per_member": 181.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.80208396911621, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8173701129083908, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_mfb-evaluation/_esam_diagnostics/esam_mfb-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4493539523791639, + "mae": 0.35018929839134216, + "r2": 0.5361924767494202, + "score": -0.4493539523791639 + }, + "val": { + "rmse": 0.46541405002003006, + "mae": 0.36193493008613586, + "r2": 0.5032421946525574, + "score": -0.46541405002003006 + }, + "test": { + "rmse": 0.4825725860549557, + "mae": 0.3769952356815338, + "r2": 0.48078668117523193, + "score": -0.4825725860549557 + } + }, + "time": "0:19:04.620800", + "diversity": { + "train": { + "mean_centered_corr": -0.02012588083744049, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05346926301717758 + }, + "val": { + "mean_centered_corr": -0.01973523572087288, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0558118112385273 + }, + "test": { + "mean_centered_corr": -0.019737333059310913, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05675049126148224 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 11 + ], + "head_selection_time": "0:00:00.027780" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_mfb-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_mfb-evaluation/0.toml new file mode 100644 index 000000000..dc867c4bc --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_mfb-evaluation/0.toml @@ -0,0 +1,56 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_mfb-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_mfb-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_mfb-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_mfb-evaluation/0/report.json new file mode 100644 index 000000000..bc42a3cd8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_mfb-evaluation/0/report.json @@ -0,0 +1,189 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9000651240348816, + "min_features_per_member": 164.0, + "mean_features_per_member": 172.8125, + "max_features_per_member": 181.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.80208396911621, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8173701129083908, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_mfb-evaluation/_esam_diagnostics/esam_mfb-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4440070475639729, + "mae": 0.34553396701812744, + "r2": 0.5471645593643188, + "score": -0.4440070475639729 + }, + "val": { + "rmse": 0.4629259699594227, + "mae": 0.3597675561904907, + "r2": 0.5085393190383911, + "score": -0.4629259699594227 + }, + "test": { + "rmse": 0.4803877661460669, + "mae": 0.3748730719089508, + "r2": 0.4854774475097656, + "score": -0.4803877661460669 + } + }, + "time": "0:19:04.620800", + "diversity": { + "train": { + "mean_centered_corr": -0.02012588083744049, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05346926301717758 + }, + "val": { + "mean_centered_corr": -0.01973523572087288, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0558118112385273 + }, + "test": { + "mean_centered_corr": -0.019737333059310913, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05675049126148224 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_mfb-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_mfb-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_mfb-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_mfb-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..18d91dc2a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_mfb-greedy-heads-evaluation/0/report.json @@ -0,0 +1,208 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9000651240348816, + "min_features_per_member": 164.0, + "mean_features_per_member": 172.8125, + "max_features_per_member": 181.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.80208396911621, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8173701129083908, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_mfb-evaluation/_esam_diagnostics/esam_mfb-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4441718165198698, + "mae": 0.3457135260105133, + "r2": 0.5468284487724304, + "score": -0.4441718165198698 + }, + "val": { + "rmse": 0.4625995219609569, + "mae": 0.3594971299171448, + "r2": 0.5092321634292603, + "score": -0.4625995219609569 + }, + "test": { + "rmse": 0.47992605869184196, + "mae": 0.37455591559410095, + "r2": 0.4864659905433655, + "score": -0.47992605869184196 + } + }, + "time": "0:19:04.620800", + "diversity": { + "train": { + "mean_centered_corr": -0.02012588083744049, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05346926301717758 + }, + "val": { + "mean_centered_corr": -0.01973523572087288, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0558118112385273 + }, + "test": { + "mean_centered_corr": -0.019737333059310913, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05675049126148224 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 11, + 24, + 25, + 5, + 3, + 27, + 17, + 4, + 18, + 1, + 24, + 25, + 20, + 8, + 14, + 30 + ], + "head_selection_time": "0:00:00.669894" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..ad4af5dc8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.001-best-head-evaluation/0/report.json @@ -0,0 +1,157 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.001-evaluation/_esam_diagnostics/esam_rho0.001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44680097340283104, + "mae": 0.3478912115097046, + "r2": 0.5414477586746216, + "score": -0.44680097340283104 + }, + "val": { + "rmse": 0.4650444270773418, + "mae": 0.36133578419685364, + "r2": 0.5040308833122253, + "score": -0.4650444270773418 + }, + "test": { + "rmse": 0.48297671022399175, + "mae": 0.37703683972358704, + "r2": 0.47991669178009033, + "score": -0.48297671022399175 + } + }, + "time": "0:15:51.200944", + "diversity": { + "train": { + "mean_centered_corr": -0.020390046760439873, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05101626738905907 + }, + "val": { + "mean_centered_corr": -0.02039702609181404, + "mean_pairwise_disagreement": NaN, + "member_std": 0.052604030817747116 + }, + "test": { + "mean_centered_corr": -0.020311973989009857, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05323263257741928 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 17 + ], + "head_selection_time": "0:00:00.029762" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.001-evaluation/0.toml new file mode 100644 index 000000000..b1552956d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.001-evaluation/0.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.001 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.001-evaluation/0/report.json new file mode 100644 index 000000000..d933df272 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.001-evaluation/0/report.json @@ -0,0 +1,153 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.001-evaluation/_esam_diagnostics/esam_rho0.001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.443936564623215, + "mae": 0.3454599678516388, + "r2": 0.5473083257675171, + "score": -0.443936564623215 + }, + "val": { + "rmse": 0.46276101981921397, + "mae": 0.35961103439331055, + "r2": 0.5088894367218018, + "score": -0.46276101981921397 + }, + "test": { + "rmse": 0.4801992110036178, + "mae": 0.3746589124202728, + "r2": 0.4858812689781189, + "score": -0.4801992110036178 + } + }, + "time": "0:15:51.200944", + "diversity": { + "train": { + "mean_centered_corr": -0.020390046760439873, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05101626738905907 + }, + "val": { + "mean_centered_corr": -0.02039702609181404, + "mean_pairwise_disagreement": NaN, + "member_std": 0.052604030817747116 + }, + "test": { + "mean_centered_corr": -0.020311973989009857, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05323263257741928 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..67847cbdd --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,168 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.001-evaluation/_esam_diagnostics/esam_rho0.001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4438539175131199, + "mae": 0.3454529643058777, + "r2": 0.5474768877029419, + "score": -0.4438539175131199 + }, + "val": { + "rmse": 0.4625151676835406, + "mae": 0.35954388976097107, + "r2": 0.5094111561775208, + "score": -0.4625151676835406 + }, + "test": { + "rmse": 0.479742369480289, + "mae": 0.37443631887435913, + "r2": 0.4868590235710144, + "score": -0.479742369480289 + } + }, + "time": "0:15:51.200944", + "diversity": { + "train": { + "mean_centered_corr": -0.020390046760439873, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05101626738905907 + }, + "val": { + "mean_centered_corr": -0.02039702609181404, + "mean_pairwise_disagreement": NaN, + "member_std": 0.052604030817747116 + }, + "test": { + "mean_centered_corr": -0.020311973989009857, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05323263257741928 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 17, + 27, + 3, + 5, + 4, + 10, + 14, + 24, + 25, + 20, + 16, + 22 + ], + "head_selection_time": "0:00:00.467943" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.0025-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.0025-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.0025-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.0025-best-head-evaluation/0/report.json new file mode 100644 index 000000000..ef69ab721 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.0025-best-head-evaluation/0/report.json @@ -0,0 +1,157 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.0025-evaluation/_esam_diagnostics/esam_rho0.0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44801757104722606, + "mae": 0.348697692155838, + "r2": 0.5389471054077148, + "score": -0.44801757104722606 + }, + "val": { + "rmse": 0.46513258351328474, + "mae": 0.36148601770401, + "r2": 0.503842830657959, + "score": -0.46513258351328474 + }, + "test": { + "rmse": 0.48253251942068065, + "mae": 0.37657663226127625, + "r2": 0.48087286949157715, + "score": -0.48253251942068065 + } + }, + "time": "0:15:54.055312", + "diversity": { + "train": { + "mean_centered_corr": -0.020338764414191246, + "mean_pairwise_disagreement": NaN, + "member_std": 0.050871677696704865 + }, + "val": { + "mean_centered_corr": -0.02030899003148079, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05245138704776764 + }, + "test": { + "mean_centered_corr": -0.020395034924149513, + "mean_pairwise_disagreement": NaN, + "member_std": 0.053137533366680145 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 7 + ], + "head_selection_time": "0:00:00.028441" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.0025-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.0025-evaluation/0.toml new file mode 100644 index 000000000..1d55eda06 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.0025-evaluation/0.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.0025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.0025-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.0025-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.0025-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.0025-evaluation/0/report.json new file mode 100644 index 000000000..7b9d04d15 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.0025-evaluation/0/report.json @@ -0,0 +1,153 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.0025-evaluation/_esam_diagnostics/esam_rho0.0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4439953515264817, + "mae": 0.34551990032196045, + "r2": 0.547188401222229, + "score": -0.4439953515264817 + }, + "val": { + "rmse": 0.4627563346148938, + "mae": 0.3595946729183197, + "r2": 0.5088993906974792, + "score": -0.4627563346148938 + }, + "test": { + "rmse": 0.4802735094812688, + "mae": 0.3747326135635376, + "r2": 0.4857221841812134, + "score": -0.4802735094812688 + } + }, + "time": "0:15:54.055312", + "diversity": { + "train": { + "mean_centered_corr": -0.020338764414191246, + "mean_pairwise_disagreement": NaN, + "member_std": 0.050871677696704865 + }, + "val": { + "mean_centered_corr": -0.02030899003148079, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05245138704776764 + }, + "test": { + "mean_centered_corr": -0.020395034924149513, + "mean_pairwise_disagreement": NaN, + "member_std": 0.053137533366680145 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.0025-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.0025-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.0025-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.0025-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..d892e337e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.0025-greedy-heads-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.0025-evaluation/_esam_diagnostics/esam_rho0.0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4440075509733049, + "mae": 0.345533162355423, + "r2": 0.5471635460853577, + "score": -0.4440075509733049 + }, + "val": { + "rmse": 0.46246732196836127, + "mae": 0.35945001244544983, + "r2": 0.5095126032829285, + "score": -0.46246732196836127 + }, + "test": { + "rmse": 0.4800169922135662, + "mae": 0.3746195435523987, + "r2": 0.48627138137817383, + "score": -0.4800169922135662 + } + }, + "time": "0:15:54.055312", + "diversity": { + "train": { + "mean_centered_corr": -0.020338764414191246, + "mean_pairwise_disagreement": NaN, + "member_std": 0.050871677696704865 + }, + "val": { + "mean_centered_corr": -0.02030899003148079, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05245138704776764 + }, + "test": { + "mean_centered_corr": -0.020395034924149513, + "mean_pairwise_disagreement": NaN, + "member_std": 0.053137533366680145 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 7, + 5, + 27, + 17, + 24, + 3, + 14, + 4, + 22, + 5, + 10, + 21, + 17, + 27, + 24, + 16, + 3, + 5, + 12, + 18, + 14, + 27, + 17 + ], + "head_selection_time": "0:00:01.091181" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..c87dbd7e3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.005-best-head-evaluation/0/report.json @@ -0,0 +1,157 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.005-evaluation/_esam_diagnostics/esam_rho0.005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4481166090239535, + "mae": 0.3489066958427429, + "r2": 0.538743257522583, + "score": -0.4481166090239535 + }, + "val": { + "rmse": 0.4651283707110853, + "mae": 0.3615894913673401, + "r2": 0.5038518309593201, + "score": -0.4651283707110853 + }, + "test": { + "rmse": 0.4823068025931435, + "mae": 0.37674346566200256, + "r2": 0.48135846853256226, + "score": -0.4823068025931435 + } + }, + "time": "0:15:51.054198", + "diversity": { + "train": { + "mean_centered_corr": -0.020383337512612343, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05032477155327797 + }, + "val": { + "mean_centered_corr": -0.02031499706208706, + "mean_pairwise_disagreement": NaN, + "member_std": 0.051892586052417755 + }, + "test": { + "mean_centered_corr": -0.02030996046960354, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05258215218782425 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 3 + ], + "head_selection_time": "0:00:00.029198" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.005-evaluation/0.toml new file mode 100644 index 000000000..ae52f3ec3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.005-evaluation/0.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.005-evaluation/0/report.json new file mode 100644 index 000000000..47f9c456b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.005-evaluation/0/report.json @@ -0,0 +1,153 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.005-evaluation/_esam_diagnostics/esam_rho0.005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44408822333443154, + "mae": 0.3455769121646881, + "r2": 0.5469989776611328, + "score": -0.44408822333443154 + }, + "val": { + "rmse": 0.4628077726618535, + "mae": 0.3596377372741699, + "r2": 0.5087901949882507, + "score": -0.4628077726618535 + }, + "test": { + "rmse": 0.48022789849772834, + "mae": 0.37472617626190186, + "r2": 0.48581981658935547, + "score": -0.48022789849772834 + } + }, + "time": "0:15:51.054198", + "diversity": { + "train": { + "mean_centered_corr": -0.020383337512612343, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05032477155327797 + }, + "val": { + "mean_centered_corr": -0.02031499706208706, + "mean_pairwise_disagreement": NaN, + "member_std": 0.051892586052417755 + }, + "test": { + "mean_centered_corr": -0.02030996046960354, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05258215218782425 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..4ca5e3f0e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,167 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.005-evaluation/_esam_diagnostics/esam_rho0.005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4439705320848652, + "mae": 0.34557339549064636, + "r2": 0.5472390651702881, + "score": -0.4439705320848652 + }, + "val": { + "rmse": 0.4625398779765968, + "mae": 0.35953423380851746, + "r2": 0.5093587040901184, + "score": -0.4625398779765968 + }, + "test": { + "rmse": 0.4797024392153251, + "mae": 0.3744461238384247, + "r2": 0.48694443702697754, + "score": -0.4797024392153251 + } + }, + "time": "0:15:51.054198", + "diversity": { + "train": { + "mean_centered_corr": -0.020383337512612343, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05032477155327797 + }, + "val": { + "mean_centered_corr": -0.02031499706208706, + "mean_pairwise_disagreement": NaN, + "member_std": 0.051892586052417755 + }, + "test": { + "mean_centered_corr": -0.02030996046960354, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05258215218782425 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 3, + 17, + 5, + 27, + 22, + 14, + 10, + 4, + 24, + 25, + 20 + ], + "head_selection_time": "0:00:00.400838" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..b24f0e085 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.01-best-head-evaluation/0/report.json @@ -0,0 +1,157 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.01-evaluation/_esam_diagnostics/esam_rho0.01-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44671700492269856, + "mae": 0.3479386866092682, + "r2": 0.5416200160980225, + "score": -0.44671700492269856 + }, + "val": { + "rmse": 0.46502524927765376, + "mae": 0.36198684573173523, + "r2": 0.504071831703186, + "score": -0.46502524927765376 + }, + "test": { + "rmse": 0.48171681453527604, + "mae": 0.3763014078140259, + "r2": 0.4826264977455139, + "score": -0.48171681453527604 + } + }, + "time": "0:15:52.573639", + "diversity": { + "train": { + "mean_centered_corr": -0.020365934818983078, + "mean_pairwise_disagreement": NaN, + "member_std": 0.049931272864341736 + }, + "val": { + "mean_centered_corr": -0.020258555188775063, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05144479125738144 + }, + "test": { + "mean_centered_corr": -0.020444629713892937, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05212138593196869 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.028008" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.01-evaluation/0.toml new file mode 100644 index 000000000..01868a998 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.01-evaluation/0.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.01 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.01-evaluation/0/report.json new file mode 100644 index 000000000..ec2147338 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.01-evaluation/0/report.json @@ -0,0 +1,153 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.01-evaluation/_esam_diagnostics/esam_rho0.01-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4441836085526797, + "mae": 0.3456646800041199, + "r2": 0.5468044281005859, + "score": -0.4441836085526797 + }, + "val": { + "rmse": 0.462806194991412, + "mae": 0.3596384525299072, + "r2": 0.5087935924530029, + "score": -0.462806194991412 + }, + "test": { + "rmse": 0.4802449022823802, + "mae": 0.3747553825378418, + "r2": 0.4857834577560425, + "score": -0.4802449022823802 + } + }, + "time": "0:15:52.573639", + "diversity": { + "train": { + "mean_centered_corr": -0.020365934818983078, + "mean_pairwise_disagreement": NaN, + "member_std": 0.049931272864341736 + }, + "val": { + "mean_centered_corr": -0.020258555188775063, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05144479125738144 + }, + "test": { + "mean_centered_corr": -0.020444629713892937, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05212138593196869 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..d77e052f8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,172 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.01-evaluation/_esam_diagnostics/esam_rho0.01-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4441221456449407, + "mae": 0.345669686794281, + "r2": 0.5469298362731934, + "score": -0.4441221456449407 + }, + "val": { + "rmse": 0.46260330683144324, + "mae": 0.35956308245658875, + "r2": 0.5092241168022156, + "score": -0.46260330683144324 + }, + "test": { + "rmse": 0.47994838230926673, + "mae": 0.37468981742858887, + "r2": 0.4864181876182556, + "score": -0.47994838230926673 + } + }, + "time": "0:15:52.573639", + "diversity": { + "train": { + "mean_centered_corr": -0.020365934818983078, + "mean_pairwise_disagreement": NaN, + "member_std": 0.049931272864341736 + }, + "val": { + "mean_centered_corr": -0.020258555188775063, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05144479125738144 + }, + "test": { + "mean_centered_corr": -0.020444629713892937, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05212138593196869 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 14, + 3, + 10, + 5, + 15, + 17, + 27, + 25, + 20, + 22, + 24, + 14, + 10, + 21, + 3 + ], + "head_selection_time": "0:00:00.639394" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_cf_fisd-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_cf_fisd-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_cf_fisd-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_cf_fisd-best-head-evaluation/0/report.json new file mode 100644 index 000000000..4d1fd8128 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_cf_fisd-best-head-evaluation/0/report.json @@ -0,0 +1,436 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9000651240348816, + "min_features_per_member": 164.0, + "mean_features_per_member": 172.8125, + "max_features_per_member": 181.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.80208396911621, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8173701129083908, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.45186522081485614, + "mae": 0.35178253054618835, + "r2": 0.5309939384460449, + "score": -0.45186522081485614 + }, + "val": { + "rmse": 0.46410829401604725, + "mae": 0.36094433069229126, + "r2": 0.5060256719589233, + "score": -0.46410829401604725 + }, + "test": { + "rmse": 0.4815192949196226, + "mae": 0.3759884238243103, + "r2": 0.48305070400238037, + "score": -0.4815192949196226 + } + }, + "time": "0:12:01.248416", + "diversity": { + "train": { + "mean_centered_corr": -0.008978656493127346, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04612109065055847 + }, + "val": { + "mean_centered_corr": -0.008577198721468449, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04788525402545929 + }, + "test": { + "mean_centered_corr": -0.008744378574192524, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04893070459365845 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 25 + ], + "head_selection_time": "0:00:00.028774" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_cf_fisd-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_cf_fisd-evaluation/0.toml new file mode 100644 index 000000000..5bd9cae8a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_cf_fisd-evaluation/0.toml @@ -0,0 +1,57 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 8 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "cooking-time" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_cf_fisd-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_cf_fisd-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_cf_fisd-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_cf_fisd-evaluation/0/report.json new file mode 100644 index 000000000..071bb8a22 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_cf_fisd-evaluation/0/report.json @@ -0,0 +1,432 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9000651240348816, + "min_features_per_member": 164.0, + "mean_features_per_member": 172.8125, + "max_features_per_member": 181.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.80208396911621, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8173701129083908, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.44850098031020863, + "mae": 0.3491051197052002, + "r2": 0.5379515886306763, + "score": -0.44850098031020863 + }, + "val": { + "rmse": 0.4633989882463895, + "mae": 0.3602413833141327, + "r2": 0.507534384727478, + "score": -0.4633989882463895 + }, + "test": { + "rmse": 0.4810100390661497, + "mae": 0.3754710555076599, + "r2": 0.48414361476898193, + "score": -0.4810100390661497 + } + }, + "time": "0:12:01.248416", + "diversity": { + "train": { + "mean_centered_corr": -0.008978656493127346, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04612109065055847 + }, + "val": { + "mean_centered_corr": -0.008577198721468449, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04788525402545929 + }, + "test": { + "mean_centered_corr": -0.008744378574192524, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04893070459365845 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_cf_fisd-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_cf_fisd-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_cf_fisd-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_cf_fisd-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..ff9be6afe --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_cf_fisd-greedy-heads-evaluation/0/report.json @@ -0,0 +1,449 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9000651240348816, + "min_features_per_member": 164.0, + "mean_features_per_member": 172.8125, + "max_features_per_member": 181.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.80208396911621, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8173701129083908, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.4473229583907692, + "mae": 0.3482147455215454, + "r2": 0.5403756499290466, + "score": -0.4473229583907692 + }, + "val": { + "rmse": 0.46284785656584665, + "mae": 0.35979634523391724, + "r2": 0.5087051391601562, + "score": -0.46284785656584665 + }, + "test": { + "rmse": 0.4803782742299631, + "mae": 0.374970942735672, + "r2": 0.48549777269363403, + "score": -0.4803782742299631 + } + }, + "time": "0:12:01.248416", + "diversity": { + "train": { + "mean_centered_corr": -0.008978656493127346, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04612109065055847 + }, + "val": { + "mean_centered_corr": -0.008577198721468449, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04788525402545929 + }, + "test": { + "mean_centered_corr": -0.008744378574192524, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04893070459365845 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 25, + 2, + 19, + 26, + 12, + 27, + 14, + 1, + 21, + 26, + 24, + 19, + 2, + 11 + ], + "head_selection_time": "0:00:00.537217" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.7-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.7-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.7-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.7-best-head-evaluation/0/report.json new file mode 100644 index 000000000..40a70ff9b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.7-best-head-evaluation/0/report.json @@ -0,0 +1,183 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.70361328125, + "min_features_per_member": 121.0, + "mean_features_per_member": 135.09375, + "max_features_per_member": 148.0, + "min_members_per_feature": 16.0, + "mean_members_per_feature": 22.515625, + "max_members_per_feature": 28.0, + "mean_pairwise_jaccard": 0.5425720205382745, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4475517856746811, + "mae": 0.34862762689590454, + "r2": 0.539905309677124, + "score": -0.4475517856746811 + }, + "val": { + "rmse": 0.46477365365967316, + "mae": 0.3614905774593353, + "r2": 0.5046082735061646, + "score": -0.46477365365967316 + }, + "test": { + "rmse": 0.48197477680705497, + "mae": 0.37628859281539917, + "r2": 0.48207223415374756, + "score": -0.48197477680705497 + } + }, + "time": "0:12:08.409861", + "diversity": { + "train": { + "mean_centered_corr": -0.019095098599791527, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06140593811869621 + }, + "val": { + "mean_centered_corr": -0.017920419573783875, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06572531163692474 + }, + "test": { + "mean_centered_corr": -0.017684392631053925, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06748230010271072 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.027824" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.7-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.7-evaluation/0.toml new file mode 100644 index 000000000..669bb05cd --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.7-evaluation/0.toml @@ -0,0 +1,46 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.7 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.7-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.7-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.7-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.7-evaluation/0/report.json new file mode 100644 index 000000000..f5894fa36 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.7-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.70361328125, + "min_features_per_member": 121.0, + "mean_features_per_member": 135.09375, + "max_features_per_member": 148.0, + "min_members_per_feature": 16.0, + "mean_members_per_feature": 22.515625, + "max_members_per_feature": 28.0, + "mean_pairwise_jaccard": 0.5425720205382745, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44401042039559846, + "mae": 0.34563449025154114, + "r2": 0.5471577048301697, + "score": -0.44401042039559846 + }, + "val": { + "rmse": 0.4641162243965573, + "mae": 0.3610197603702545, + "r2": 0.5060087442398071, + "score": -0.4641162243965573 + }, + "test": { + "rmse": 0.4815966695140456, + "mae": 0.3760018050670624, + "r2": 0.48288458585739136, + "score": -0.4815966695140456 + } + }, + "time": "0:12:08.409861", + "diversity": { + "train": { + "mean_centered_corr": -0.019095098599791527, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06140593811869621 + }, + "val": { + "mean_centered_corr": -0.017920419573783875, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06572531163692474 + }, + "test": { + "mean_centered_corr": -0.017684392631053925, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06748230010271072 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.7-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.7-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.7-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.7-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..3a48b73af --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.7-greedy-heads-evaluation/0/report.json @@ -0,0 +1,197 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.70361328125, + "min_features_per_member": 121.0, + "mean_features_per_member": 135.09375, + "max_features_per_member": 148.0, + "min_members_per_feature": 16.0, + "mean_members_per_feature": 22.515625, + "max_members_per_feature": 28.0, + "mean_pairwise_jaccard": 0.5425720205382745, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44399968094212866, + "mae": 0.3455778956413269, + "r2": 0.547179639339447, + "score": -0.44399968094212866 + }, + "val": { + "rmse": 0.46249551448708903, + "mae": 0.35950425267219543, + "r2": 0.5094528198242188, + "score": -0.46249551448708903 + }, + "test": { + "rmse": 0.47980768580603567, + "mae": 0.37452432513237, + "r2": 0.4867192506790161, + "score": -0.47980768580603567 + } + }, + "time": "0:12:08.409861", + "diversity": { + "train": { + "mean_centered_corr": -0.019095098599791527, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06140593811869621 + }, + "val": { + "mean_centered_corr": -0.017920419573783875, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06572531163692474 + }, + "test": { + "mean_centered_corr": -0.017684392631053925, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06748230010271072 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 4, + 7, + 24, + 25, + 11, + 9, + 31, + 12, + 27, + 14, + 22, + 24, + 5, + 20, + 25 + ], + "head_selection_time": "0:00:00.591913" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.8-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.8-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.8-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.8-best-head-evaluation/0/report.json new file mode 100644 index 000000000..50ed2f8b2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.8-best-head-evaluation/0/report.json @@ -0,0 +1,183 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.8019205927848816, + "min_features_per_member": 141.0, + "mean_features_per_member": 153.96875, + "max_features_per_member": 167.0, + "min_members_per_feature": 21.0, + "mean_members_per_feature": 25.66145896911621, + "max_members_per_feature": 31.0, + "mean_pairwise_jaccard": 0.6682771134235742, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44690438220657625, + "mae": 0.3479042053222656, + "r2": 0.5412354469299316, + "score": -0.44690438220657625 + }, + "val": { + "rmse": 0.4652213638497702, + "mae": 0.36172589659690857, + "r2": 0.5036534070968628, + "score": -0.4652213638497702 + }, + "test": { + "rmse": 0.48343941871247514, + "mae": 0.37721583247184753, + "r2": 0.4789196848869324, + "score": -0.48343941871247514 + } + }, + "time": "0:12:07.675037", + "diversity": { + "train": { + "mean_centered_corr": -0.019306395202875137, + "mean_pairwise_disagreement": NaN, + "member_std": 0.057879555970430374 + }, + "val": { + "mean_centered_corr": -0.01832164078950882, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06112360581755638 + }, + "test": { + "mean_centered_corr": -0.01794510707259178, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06266573071479797 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 19 + ], + "head_selection_time": "0:00:00.027886" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.8-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.8-evaluation/0.toml new file mode 100644 index 000000000..83cf7dea9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.8-evaluation/0.toml @@ -0,0 +1,46 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.8 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.8-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.8-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.8-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.8-evaluation/0/report.json new file mode 100644 index 000000000..835fe9a93 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.8-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.8019205927848816, + "min_features_per_member": 141.0, + "mean_features_per_member": 153.96875, + "max_features_per_member": 167.0, + "min_members_per_feature": 21.0, + "mean_members_per_feature": 25.66145896911621, + "max_members_per_feature": 31.0, + "mean_pairwise_jaccard": 0.6682771134235742, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4439757847261177, + "mae": 0.34554752707481384, + "r2": 0.5472283363342285, + "score": -0.4439757847261177 + }, + "val": { + "rmse": 0.4634085546239816, + "mae": 0.36031344532966614, + "r2": 0.5075141191482544, + "score": -0.4634085546239816 + }, + "test": { + "rmse": 0.4808915146871742, + "mae": 0.37534573674201965, + "r2": 0.4843977689743042, + "score": -0.4808915146871742 + } + }, + "time": "0:12:07.675037", + "diversity": { + "train": { + "mean_centered_corr": -0.019306395202875137, + "mean_pairwise_disagreement": NaN, + "member_std": 0.057879555970430374 + }, + "val": { + "mean_centered_corr": -0.01832164078950882, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06112360581755638 + }, + "test": { + "mean_centered_corr": -0.01794510707259178, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06266573071479797 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.8-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.8-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.8-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.8-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..c9f5dc71a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.8-greedy-heads-evaluation/0/report.json @@ -0,0 +1,193 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.8019205927848816, + "min_features_per_member": 141.0, + "mean_features_per_member": 153.96875, + "max_features_per_member": 167.0, + "min_members_per_feature": 21.0, + "mean_members_per_feature": 25.66145896911621, + "max_members_per_feature": 31.0, + "mean_pairwise_jaccard": 0.6682771134235742, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44390037903332324, + "mae": 0.34550046920776367, + "r2": 0.547382116317749, + "score": -0.44390037903332324 + }, + "val": { + "rmse": 0.4624822561623469, + "mae": 0.35955899953842163, + "r2": 0.5094809532165527, + "score": -0.4624822561623469 + }, + "test": { + "rmse": 0.4798057758257153, + "mae": 0.37443193793296814, + "r2": 0.4867233633995056, + "score": -0.4798057758257153 + } + }, + "time": "0:12:07.675037", + "diversity": { + "train": { + "mean_centered_corr": -0.019306395202875137, + "mean_pairwise_disagreement": NaN, + "member_std": 0.057879555970430374 + }, + "val": { + "mean_centered_corr": -0.01832164078950882, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06112360581755638 + }, + "test": { + "mean_centered_corr": -0.01794510707259178, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06266573071479797 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 19, + 27, + 17, + 24, + 25, + 20, + 22, + 4, + 14, + 7, + 5 + ], + "head_selection_time": "0:00:00.407380" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.9-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.9-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.9-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.9-best-head-evaluation/0/report.json new file mode 100644 index 000000000..b5e7724cd --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.9-best-head-evaluation/0/report.json @@ -0,0 +1,183 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9000651240348816, + "min_features_per_member": 164.0, + "mean_features_per_member": 172.8125, + "max_features_per_member": 181.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.80208396911621, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8173701129083908, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44680861066629424, + "mae": 0.3479525148868561, + "r2": 0.5414320230484009, + "score": -0.44680861066629424 + }, + "val": { + "rmse": 0.465403548325863, + "mae": 0.3621026575565338, + "r2": 0.5032646059989929, + "score": -0.465403548325863 + }, + "test": { + "rmse": 0.48311284423774803, + "mae": 0.377098023891449, + "r2": 0.4796234369277954, + "score": -0.48311284423774803 + } + }, + "time": "0:12:04.046616", + "diversity": { + "train": { + "mean_centered_corr": -0.020108899101614952, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05420300364494324 + }, + "val": { + "mean_centered_corr": -0.01979835145175457, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05661233514547348 + }, + "test": { + "mean_centered_corr": -0.01982581429183483, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05757860466837883 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 17 + ], + "head_selection_time": "0:00:00.027733" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.9-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.9-evaluation/0.toml new file mode 100644 index 000000000..349df099f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.9-evaluation/0.toml @@ -0,0 +1,46 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.9-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.9-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.9-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.9-evaluation/0/report.json new file mode 100644 index 000000000..ecbe78388 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.9-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9000651240348816, + "min_features_per_member": 164.0, + "mean_features_per_member": 172.8125, + "max_features_per_member": 181.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.80208396911621, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8173701129083908, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4438062088250124, + "mae": 0.34538015723228455, + "r2": 0.5475741624832153, + "score": -0.4438062088250124 + }, + "val": { + "rmse": 0.46290981075910853, + "mae": 0.3597574234008789, + "r2": 0.508573591709137, + "score": -0.46290981075910853 + }, + "test": { + "rmse": 0.480415480847424, + "mae": 0.37490591406822205, + "r2": 0.48541808128356934, + "score": -0.480415480847424 + } + }, + "time": "0:12:04.046616", + "diversity": { + "train": { + "mean_centered_corr": -0.020108899101614952, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05420300364494324 + }, + "val": { + "mean_centered_corr": -0.01979835145175457, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05661233514547348 + }, + "test": { + "mean_centered_corr": -0.01982581429183483, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05757860466837883 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.9-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.9-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.9-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.9-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..4888bef62 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.9-greedy-heads-evaluation/0/report.json @@ -0,0 +1,193 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9000651240348816, + "min_features_per_member": 164.0, + "mean_features_per_member": 172.8125, + "max_features_per_member": 181.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.80208396911621, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8173701129083908, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4439127657129147, + "mae": 0.3454906642436981, + "r2": 0.5473568439483643, + "score": -0.4439127657129147 + }, + "val": { + "rmse": 0.4625593198933149, + "mae": 0.35958266258239746, + "r2": 0.5093174576759338, + "score": -0.4625593198933149 + }, + "test": { + "rmse": 0.479899744054174, + "mae": 0.37457266449928284, + "r2": 0.4865223169326782, + "score": -0.479899744054174 + } + }, + "time": "0:12:04.046616", + "diversity": { + "train": { + "mean_centered_corr": -0.020108899101614952, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05420300364494324 + }, + "val": { + "mean_centered_corr": -0.01979835145175457, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05661233514547348 + }, + "test": { + "mean_centered_corr": -0.01982581429183483, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05757860466837883 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 17, + 14, + 11, + 24, + 25, + 5, + 4, + 27, + 20, + 1, + 9 + ], + "head_selection_time": "0:00:00.405579" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.95-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.95-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.95-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.95-best-head-evaluation/0/report.json new file mode 100644 index 000000000..08801eb88 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.95-best-head-evaluation/0/report.json @@ -0,0 +1,183 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9480794072151184, + "min_features_per_member": 176.0, + "mean_features_per_member": 182.03125, + "max_features_per_member": 188.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.33854103088379, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.900785048828594, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4475263477006594, + "mae": 0.34830242395401, + "r2": 0.5399576425552368, + "score": -0.4475263477006594 + }, + "val": { + "rmse": 0.4649031464912309, + "mae": 0.36123916506767273, + "r2": 0.5043321847915649, + "score": -0.4649031464912309 + }, + "test": { + "rmse": 0.4828450127182003, + "mae": 0.376803457736969, + "r2": 0.48020023107528687, + "score": -0.4828450127182003 + } + }, + "time": "0:12:04.148232", + "diversity": { + "train": { + "mean_centered_corr": -0.020304471254348755, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05282822996377945 + }, + "val": { + "mean_centered_corr": -0.02010956034064293, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0548769012093544 + }, + "test": { + "mean_centered_corr": -0.020047690719366074, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05574715510010719 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 7 + ], + "head_selection_time": "0:00:00.027586" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.95-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.95-evaluation/0.toml new file mode 100644 index 000000000..3571f95da --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.95-evaluation/0.toml @@ -0,0 +1,46 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.95 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.95-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.95-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.95-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.95-evaluation/0/report.json new file mode 100644 index 000000000..f97797710 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.95-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9480794072151184, + "min_features_per_member": 176.0, + "mean_features_per_member": 182.03125, + "max_features_per_member": 188.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.33854103088379, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.900785048828594, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.443937454120516, + "mae": 0.34546512365341187, + "r2": 0.5473065376281738, + "score": -0.443937454120516 + }, + "val": { + "rmse": 0.46283832689198484, + "mae": 0.35965093970298767, + "r2": 0.5087253451347351, + "score": -0.46283832689198484 + }, + "test": { + "rmse": 0.48036774294670015, + "mae": 0.3747703433036804, + "r2": 0.48552030324935913, + "score": -0.48036774294670015 + } + }, + "time": "0:12:04.148232", + "diversity": { + "train": { + "mean_centered_corr": -0.020304471254348755, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05282822996377945 + }, + "val": { + "mean_centered_corr": -0.02010956034064293, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0548769012093544 + }, + "test": { + "mean_centered_corr": -0.020047690719366074, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05574715510010719 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.95-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.95-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.95-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.95-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..acdf05465 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.95-greedy-heads-evaluation/0/report.json @@ -0,0 +1,197 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9480794072151184, + "min_features_per_member": 176.0, + "mean_features_per_member": 182.03125, + "max_features_per_member": 188.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.33854103088379, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.900785048828594, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.444191827565372, + "mae": 0.34573376178741455, + "r2": 0.5467876195907593, + "score": -0.444191827565372 + }, + "val": { + "rmse": 0.46250350473983365, + "mae": 0.35950493812561035, + "r2": 0.5094358921051025, + "score": -0.46250350473983365 + }, + "test": { + "rmse": 0.4799452154615134, + "mae": 0.37449464201927185, + "r2": 0.48642498254776, + "score": -0.4799452154615134 + } + }, + "time": "0:12:04.148232", + "diversity": { + "train": { + "mean_centered_corr": -0.020304471254348755, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05282822996377945 + }, + "val": { + "mean_centered_corr": -0.02010956034064293, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0548769012093544 + }, + "test": { + "mean_centered_corr": -0.020047690719366074, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05574715510010719 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 7, + 14, + 17, + 20, + 3, + 24, + 22, + 1, + 5, + 9, + 28, + 14, + 11, + 17, + 24 + ], + "head_selection_time": "0:00:00.596709" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_cf_fisd-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_cf_fisd-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_cf_fisd-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_cf_fisd-best-head-evaluation/0/report.json new file mode 100644 index 000000000..439dc1c47 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_cf_fisd-best-head-evaluation/0/report.json @@ -0,0 +1,409 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.4519491065266232, + "mae": 0.3517553210258484, + "r2": 0.5308197736740112, + "score": -0.4519491065266232 + }, + "val": { + "rmse": 0.4638249749501882, + "mae": 0.36049750447273254, + "r2": 0.5066285729408264, + "score": -0.4638249749501882 + }, + "test": { + "rmse": 0.4819157837704579, + "mae": 0.3760508596897125, + "r2": 0.4821990132331848, + "score": -0.4819157837704579 + } + }, + "time": "0:21:02.492076", + "diversity": { + "train": { + "mean_centered_corr": -0.004915067460387945, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04214517027139664 + }, + "val": { + "mean_centered_corr": -0.00553104979917407, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04312119632959366 + }, + "test": { + "mean_centered_corr": -0.005271091125905514, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04374181479215622 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 31 + ], + "head_selection_time": "0:00:00.030941" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_cf_fisd-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_cf_fisd-evaluation/0.toml new file mode 100644 index 000000000..5233fa3a6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_cf_fisd-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "cooking-time" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_cf_fisd-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_cf_fisd-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_cf_fisd-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_cf_fisd-evaluation/0/report.json new file mode 100644 index 000000000..d9a8d49f6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_cf_fisd-evaluation/0/report.json @@ -0,0 +1,405 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.44810794656884273, + "mae": 0.3487721383571625, + "r2": 0.5387611389160156, + "score": -0.44810794656884273 + }, + "val": { + "rmse": 0.46314488441622087, + "mae": 0.36001190543174744, + "r2": 0.5080743432044983, + "score": -0.46314488441622087 + }, + "test": { + "rmse": 0.48088551875549623, + "mae": 0.37529256939888, + "r2": 0.4844106435775757, + "score": -0.48088551875549623 + } + }, + "time": "0:21:02.492076", + "diversity": { + "train": { + "mean_centered_corr": -0.004915067460387945, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04214517027139664 + }, + "val": { + "mean_centered_corr": -0.00553104979917407, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04312119632959366 + }, + "test": { + "mean_centered_corr": -0.005271091125905514, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04374181479215622 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_cf_fisd-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_cf_fisd-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_cf_fisd-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_cf_fisd-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..788ac6a5d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_cf_fisd-greedy-heads-evaluation/0/report.json @@ -0,0 +1,413 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.44774267417605834, + "mae": 0.3486373722553253, + "r2": 0.5395127534866333, + "score": -0.44774267417605834 + }, + "val": { + "rmse": 0.4627888241622746, + "mae": 0.35998469591140747, + "r2": 0.5088304281234741, + "score": -0.4627888241622746 + }, + "test": { + "rmse": 0.480810230139191, + "mae": 0.3753969371318817, + "r2": 0.48457205295562744, + "score": -0.480810230139191 + } + }, + "time": "0:21:02.492076", + "diversity": { + "train": { + "mean_centered_corr": -0.004915067460387945, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04214517027139664 + }, + "val": { + "mean_centered_corr": -0.00553104979917407, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04312119632959366 + }, + "test": { + "mean_centered_corr": -0.005271091125905514, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04374181479215622 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 31, + 14, + 2, + 24, + 21 + ], + "head_selection_time": "0:00:00.181665" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_esam-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_esam-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_esam-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_esam-best-head-evaluation/0/report.json new file mode 100644 index 000000000..f1c458896 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_esam-best-head-evaluation/0/report.json @@ -0,0 +1,166 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_esam-evaluation/_esam_diagnostics/rla_esam-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44621279542701975, + "mae": 0.3473000228404999, + "r2": 0.5426542162895203, + "score": -0.44621279542701975 + }, + "val": { + "rmse": 0.46492255368357066, + "mae": 0.36141437292099, + "r2": 0.5042908191680908, + "score": -0.46492255368357066 + }, + "test": { + "rmse": 0.4825495500767423, + "mae": 0.37668848037719727, + "r2": 0.4808362126350403, + "score": -0.4825495500767423 + } + }, + "time": "0:37:31.095542", + "diversity": { + "train": { + "mean_centered_corr": -0.02036323957145214, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05055415257811546 + }, + "val": { + "mean_centered_corr": -0.02048061601817608, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05214421823620796 + }, + "test": { + "mean_centered_corr": -0.020323824137449265, + "mean_pairwise_disagreement": NaN, + "member_std": 0.052737440913915634 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 14 + ], + "head_selection_time": "0:00:00.027759" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_esam-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_esam-evaluation/0.toml new file mode 100644 index 000000000..0b28045f1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_esam-evaluation/0.toml @@ -0,0 +1,50 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_esam-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_esam-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_esam-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_esam-evaluation/0/report.json new file mode 100644 index 000000000..9520ae39a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_esam-evaluation/0/report.json @@ -0,0 +1,162 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_esam-evaluation/_esam_diagnostics/rla_esam-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4437324533888703, + "mae": 0.3453030586242676, + "r2": 0.5477245450019836, + "score": -0.4437324533888703 + }, + "val": { + "rmse": 0.4627353230636857, + "mae": 0.35955438017845154, + "r2": 0.5089439749717712, + "score": -0.4627353230636857 + }, + "test": { + "rmse": 0.4803077303885999, + "mae": 0.3747372627258301, + "r2": 0.48564887046813965, + "score": -0.4803077303885999 + } + }, + "time": "0:37:31.095542", + "diversity": { + "train": { + "mean_centered_corr": -0.02036323957145214, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05055415257811546 + }, + "val": { + "mean_centered_corr": -0.02048061601817608, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05214421823620796 + }, + "test": { + "mean_centered_corr": -0.020323824137449265, + "mean_pairwise_disagreement": NaN, + "member_std": 0.052737440913915634 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_esam-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_esam-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_esam-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_esam-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..03c3c5cda --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_esam-greedy-heads-evaluation/0/report.json @@ -0,0 +1,178 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_esam-evaluation/_esam_diagnostics/rla_esam-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4439349534537547, + "mae": 0.3455897867679596, + "r2": 0.5473116636276245, + "score": -0.4439349534537547 + }, + "val": { + "rmse": 0.4624234187671886, + "mae": 0.3594515919685364, + "r2": 0.5096057653427124, + "score": -0.4624234187671886 + }, + "test": { + "rmse": 0.4800586966547807, + "mae": 0.37484052777290344, + "r2": 0.4861820936203003, + "score": -0.4800586966547807 + } + }, + "time": "0:37:31.095542", + "diversity": { + "train": { + "mean_centered_corr": -0.02036323957145214, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05055415257811546 + }, + "val": { + "mean_centered_corr": -0.02048061601817608, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05214421823620796 + }, + "test": { + "mean_centered_corr": -0.020323824137449265, + "mean_pairwise_disagreement": NaN, + "member_std": 0.052737440913915634 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 14, + 3, + 17, + 24, + 5, + 10, + 4, + 27, + 31, + 7, + 17, + 3, + 24 + ], + "head_selection_time": "0:00:00.494060" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_mfb-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_mfb-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_mfb-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_mfb-best-head-evaluation/0/report.json new file mode 100644 index 000000000..fd6ab34e6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_mfb-best-head-evaluation/0/report.json @@ -0,0 +1,192 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9000651240348816, + "min_features_per_member": 164.0, + "mean_features_per_member": 172.8125, + "max_features_per_member": 181.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.80208396911621, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8173701129083908, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44878779762176807, + "mae": 0.34966716170310974, + "r2": 0.5373604893684387, + "score": -0.44878779762176807 + }, + "val": { + "rmse": 0.46542866555060514, + "mae": 0.36194679141044617, + "r2": 0.5032110214233398, + "score": -0.46542866555060514 + }, + "test": { + "rmse": 0.48315113561982953, + "mae": 0.3771289885044098, + "r2": 0.47954094409942627, + "score": -0.48315113561982953 + } + }, + "time": "0:26:00.632881", + "diversity": { + "train": { + "mean_centered_corr": -0.02016231045126915, + "mean_pairwise_disagreement": NaN, + "member_std": 0.054188016802072525 + }, + "val": { + "mean_centered_corr": -0.019777867943048477, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05651267245411873 + }, + "test": { + "mean_centered_corr": -0.019633809104561806, + "mean_pairwise_disagreement": NaN, + "member_std": 0.057482462376356125 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 11 + ], + "head_selection_time": "0:00:00.028172" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_mfb-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_mfb-evaluation/0.toml new file mode 100644 index 000000000..704e95be9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_mfb-evaluation/0.toml @@ -0,0 +1,53 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_mfb-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_mfb-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_mfb-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_mfb-evaluation/0/report.json new file mode 100644 index 000000000..65cbfe41a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_mfb-evaluation/0/report.json @@ -0,0 +1,188 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9000651240348816, + "min_features_per_member": 164.0, + "mean_features_per_member": 172.8125, + "max_features_per_member": 181.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.80208396911621, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8173701129083908, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44355369712218456, + "mae": 0.34518396854400635, + "r2": 0.5480888485908508, + "score": -0.44355369712218456 + }, + "val": { + "rmse": 0.4628964355361592, + "mae": 0.3597174882888794, + "r2": 0.5086020231246948, + "score": -0.4628964355361592 + }, + "test": { + "rmse": 0.4804232040775468, + "mae": 0.3748829662799835, + "r2": 0.4854015111923218, + "score": -0.4804232040775468 + } + }, + "time": "0:26:00.632881", + "diversity": { + "train": { + "mean_centered_corr": -0.02016231045126915, + "mean_pairwise_disagreement": NaN, + "member_std": 0.054188016802072525 + }, + "val": { + "mean_centered_corr": -0.019777867943048477, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05651267245411873 + }, + "test": { + "mean_centered_corr": -0.019633809104561806, + "mean_pairwise_disagreement": NaN, + "member_std": 0.057482462376356125 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_mfb-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_mfb-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_mfb-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_mfb-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..3a9a313cf --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_mfb-greedy-heads-evaluation/0/report.json @@ -0,0 +1,207 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9000651240348816, + "min_features_per_member": 164.0, + "mean_features_per_member": 172.8125, + "max_features_per_member": 181.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.80208396911621, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8173701129083908, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44370132233029524, + "mae": 0.3453197777271271, + "r2": 0.5477880239486694, + "score": -0.44370132233029524 + }, + "val": { + "rmse": 0.46255113731378883, + "mae": 0.35945776104927063, + "r2": 0.5093348026275635, + "score": -0.46255113731378883 + }, + "test": { + "rmse": 0.4799569823688197, + "mae": 0.3745240867137909, + "r2": 0.4863998293876648, + "score": -0.4799569823688197 + } + }, + "time": "0:26:00.632881", + "diversity": { + "train": { + "mean_centered_corr": -0.02016231045126915, + "mean_pairwise_disagreement": NaN, + "member_std": 0.054188016802072525 + }, + "val": { + "mean_centered_corr": -0.019777867943048477, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05651267245411873 + }, + "test": { + "mean_centered_corr": -0.019633809104561806, + "mean_pairwise_disagreement": NaN, + "member_std": 0.057482462376356125 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 11, + 24, + 9, + 17, + 14, + 5, + 25, + 4, + 3, + 1, + 24, + 30, + 27, + 8, + 17, + 28 + ], + "head_selection_time": "0:00:00.648503" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.0001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.0001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.0001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.0001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..9e3525e8a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.0001-best-head-evaluation/0/report.json @@ -0,0 +1,156 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1520160, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4464251521524547, + "mae": 0.347421795129776, + "r2": 0.542218804359436, + "score": -0.4464251521524547 + }, + "val": { + "rmse": 0.465058894016931, + "mae": 0.3617338240146637, + "r2": 0.5040000677108765, + "score": -0.465058894016931 + }, + "test": { + "rmse": 0.48247512338182996, + "mae": 0.3766128718852997, + "r2": 0.48099637031555176, + "score": -0.48247512338182996 + } + }, + "time": "0:14:45.597272", + "diversity": { + "train": { + "mean_centered_corr": -0.020377950742840767, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05120374634861946 + }, + "val": { + "mean_centered_corr": -0.02035089209675789, + "mean_pairwise_disagreement": NaN, + "member_std": 0.052788279950618744 + }, + "test": { + "mean_centered_corr": -0.02027548849582672, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0534634031355381 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 14 + ], + "head_selection_time": "0:00:00.027776" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.0001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.0001-evaluation/0.toml new file mode 100644 index 000000000..7ba625753 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.0001-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.0001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.0001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.0001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.0001-evaluation/0/report.json new file mode 100644 index 000000000..40a403eab --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.0001-evaluation/0/report.json @@ -0,0 +1,152 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1520160, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4437718090554473, + "mae": 0.3453312814235687, + "r2": 0.5476442575454712, + "score": -0.4437718090554473 + }, + "val": { + "rmse": 0.4627581378606596, + "mae": 0.3596029281616211, + "r2": 0.5088955163955688, + "score": -0.4627581378606596 + }, + "test": { + "rmse": 0.4802975698569232, + "mae": 0.37475284934043884, + "r2": 0.48567062616348267, + "score": -0.4802975698569232 + } + }, + "time": "0:14:45.597272", + "diversity": { + "train": { + "mean_centered_corr": -0.020377950742840767, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05120374634861946 + }, + "val": { + "mean_centered_corr": -0.02035089209675789, + "mean_pairwise_disagreement": NaN, + "member_std": 0.052788279950618744 + }, + "test": { + "mean_centered_corr": -0.02027548849582672, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0534634031355381 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.0001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.0001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.0001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.0001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..0eb3076b7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.0001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,165 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1520160, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4437838635485539, + "mae": 0.34541016817092896, + "r2": 0.5476197004318237, + "score": -0.4437838635485539 + }, + "val": { + "rmse": 0.4624788086134533, + "mae": 0.3595048785209656, + "r2": 0.5094882845878601, + "score": -0.4624788086134533 + }, + "test": { + "rmse": 0.4800547545217561, + "mae": 0.3747299015522003, + "r2": 0.4861905574798584, + "score": -0.4800547545217561 + } + }, + "time": "0:14:45.597272", + "diversity": { + "train": { + "mean_centered_corr": -0.020377950742840767, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05120374634861946 + }, + "val": { + "mean_centered_corr": -0.02035089209675789, + "mean_pairwise_disagreement": NaN, + "member_std": 0.052788279950618744 + }, + "test": { + "mean_centered_corr": -0.02027548849582672, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0534634031355381 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 14, + 3, + 17, + 5, + 27, + 24, + 22, + 10, + 16, + 15 + ], + "head_selection_time": "0:00:00.358048" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..5890a5a42 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.001-best-head-evaluation/0/report.json @@ -0,0 +1,156 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1520160, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4462822343900781, + "mae": 0.347369521856308, + "r2": 0.5425118207931519, + "score": -0.4462822343900781 + }, + "val": { + "rmse": 0.465139054815724, + "mae": 0.36169102787971497, + "r2": 0.5038290619850159, + "score": -0.465139054815724 + }, + "test": { + "rmse": 0.48220568612119014, + "mae": 0.3766040503978729, + "r2": 0.4815758466720581, + "score": -0.48220568612119014 + } + }, + "time": "0:14:46.572437", + "diversity": { + "train": { + "mean_centered_corr": -0.02044508419930935, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05110829323530197 + }, + "val": { + "mean_centered_corr": -0.020311282947659492, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05272478237748146 + }, + "test": { + "mean_centered_corr": -0.0204484686255455, + "mean_pairwise_disagreement": NaN, + "member_std": 0.053339943289756775 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 14 + ], + "head_selection_time": "0:00:00.028154" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.001-evaluation/0.toml new file mode 100644 index 000000000..a07ce792d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.001-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.001-evaluation/0/report.json new file mode 100644 index 000000000..ba5555ef9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.001-evaluation/0/report.json @@ -0,0 +1,152 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1520160, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4437945578595596, + "mae": 0.34535884857177734, + "r2": 0.5475978851318359, + "score": -0.4437945578595596 + }, + "val": { + "rmse": 0.462741360963445, + "mae": 0.35957956314086914, + "r2": 0.5089311599731445, + "score": -0.462741360963445 + }, + "test": { + "rmse": 0.4802330803717953, + "mae": 0.374697744846344, + "r2": 0.48580873012542725, + "score": -0.4802330803717953 + } + }, + "time": "0:14:46.572437", + "diversity": { + "train": { + "mean_centered_corr": -0.02044508419930935, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05110829323530197 + }, + "val": { + "mean_centered_corr": -0.020311282947659492, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05272478237748146 + }, + "test": { + "mean_centered_corr": -0.0204484686255455, + "mean_pairwise_disagreement": NaN, + "member_std": 0.053339943289756775 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..0ff438b7e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,174 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1520160, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.443731832132361, + "mae": 0.3453916907310486, + "r2": 0.5477257966995239, + "score": -0.443731832132361 + }, + "val": { + "rmse": 0.4624384832710735, + "mae": 0.3594527840614319, + "r2": 0.5095738172531128, + "score": -0.4624384832710735 + }, + "test": { + "rmse": 0.47998788853107577, + "mae": 0.37460222840309143, + "r2": 0.4863336682319641, + "score": -0.47998788853107577 + } + }, + "time": "0:14:46.572437", + "diversity": { + "train": { + "mean_centered_corr": -0.02044508419930935, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05110829323530197 + }, + "val": { + "mean_centered_corr": -0.020311282947659492, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05272478237748146 + }, + "test": { + "mean_centered_corr": -0.0204484686255455, + "mean_pairwise_disagreement": NaN, + "member_std": 0.053339943289756775 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 14, + 3, + 5, + 17, + 22, + 24, + 27, + 10, + 18, + 6, + 5, + 4, + 17, + 3, + 21, + 24, + 22, + 5, + 27 + ], + "head_selection_time": "0:00:00.829666" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise1e-05-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise1e-05-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise1e-05-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise1e-05-best-head-evaluation/0/report.json new file mode 100644 index 000000000..639bab022 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise1e-05-best-head-evaluation/0/report.json @@ -0,0 +1,156 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1520160, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44786020595828924, + "mae": 0.348699688911438, + "r2": 0.5392709970474243, + "score": -0.44786020595828924 + }, + "val": { + "rmse": 0.4650147388029769, + "mae": 0.36139675974845886, + "r2": 0.5040942430496216, + "score": -0.4650147388029769 + }, + "test": { + "rmse": 0.4822676099238603, + "mae": 0.37660688161849976, + "r2": 0.4814426898956299, + "score": -0.4822676099238603 + } + }, + "time": "0:14:47.284485", + "diversity": { + "train": { + "mean_centered_corr": -0.020345786586403847, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05121167004108429 + }, + "val": { + "mean_centered_corr": -0.02022434212267399, + "mean_pairwise_disagreement": NaN, + "member_std": 0.052821069955825806 + }, + "test": { + "mean_centered_corr": -0.020434435456991196, + "mean_pairwise_disagreement": NaN, + "member_std": 0.053462423384189606 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 3 + ], + "head_selection_time": "0:00:00.028162" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise1e-05-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise1e-05-evaluation/0.toml new file mode 100644 index 000000000..cb6260a3f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise1e-05-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise1e-05-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise1e-05-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise1e-05-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise1e-05-evaluation/0/report.json new file mode 100644 index 000000000..b7e742362 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise1e-05-evaluation/0/report.json @@ -0,0 +1,152 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1520160, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44380323735423477, + "mae": 0.3453722298145294, + "r2": 0.5475802421569824, + "score": -0.44380323735423477 + }, + "val": { + "rmse": 0.46274171518445095, + "mae": 0.35957109928131104, + "r2": 0.5089304447174072, + "score": -0.46274171518445095 + }, + "test": { + "rmse": 0.48025462954174786, + "mae": 0.3747461140155792, + "r2": 0.4857625961303711, + "score": -0.48025462954174786 + } + }, + "time": "0:14:47.284485", + "diversity": { + "train": { + "mean_centered_corr": -0.020345786586403847, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05121167004108429 + }, + "val": { + "mean_centered_corr": -0.02022434212267399, + "mean_pairwise_disagreement": NaN, + "member_std": 0.052821069955825806 + }, + "test": { + "mean_centered_corr": -0.020434435456991196, + "mean_pairwise_disagreement": NaN, + "member_std": 0.053462423384189606 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise1e-05-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise1e-05-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise1e-05-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise1e-05-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..5816ea3be --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise1e-05-greedy-heads-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1520160, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4437128917866993, + "mae": 0.34536486864089966, + "r2": 0.5477644205093384, + "score": -0.4437128917866993 + }, + "val": { + "rmse": 0.46249997680251603, + "mae": 0.3594898581504822, + "r2": 0.5094433426856995, + "score": -0.46249997680251603 + }, + "test": { + "rmse": 0.47995778958671514, + "mae": 0.37462249398231506, + "r2": 0.4863981008529663, + "score": -0.47995778958671514 + } + }, + "time": "0:14:47.284485", + "diversity": { + "train": { + "mean_centered_corr": -0.020345786586403847, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05121167004108429 + }, + "val": { + "mean_centered_corr": -0.02022434212267399, + "mean_pairwise_disagreement": NaN, + "member_std": 0.052821069955825806 + }, + "test": { + "mean_centered_corr": -0.020434435456991196, + "mean_pairwise_disagreement": NaN, + "member_std": 0.053462423384189606 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 3, + 17, + 27, + 5, + 4, + 14, + 10, + 21, + 22, + 16, + 24, + 31, + 20, + 27, + 17, + 5, + 9, + 10, + 6, + 14 + ], + "head_selection_time": "0:00:00.909476" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.0001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.0001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.0001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.0001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..9ac54fdad --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.0001-best-head-evaluation/0/report.json @@ -0,0 +1,156 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44760284038640463, + "mae": 0.3484247922897339, + "r2": 0.5398002862930298, + "score": -0.44760284038640463 + }, + "val": { + "rmse": 0.4651183431272534, + "mae": 0.3614919185638428, + "r2": 0.5038732290267944, + "score": -0.4651183431272534 + }, + "test": { + "rmse": 0.48213882482447107, + "mae": 0.37617027759552, + "r2": 0.48171961307525635, + "score": -0.48213882482447107 + } + }, + "time": "0:23:02.905011", + "diversity": { + "train": { + "mean_centered_corr": -0.020404240116477013, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0513404980301857 + }, + "val": { + "mean_centered_corr": -0.02041091024875641, + "mean_pairwise_disagreement": NaN, + "member_std": 0.052927520126104355 + }, + "test": { + "mean_centered_corr": -0.020323792472481728, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05353504791855812 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 7 + ], + "head_selection_time": "0:00:00.027779" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.0001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.0001-evaluation/0.toml new file mode 100644 index 000000000..17b62a97a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.0001-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.0001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.0001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.0001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.0001-evaluation/0/report.json new file mode 100644 index 000000000..0df041ca5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.0001-evaluation/0/report.json @@ -0,0 +1,152 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4435648840971451, + "mae": 0.34517785906791687, + "r2": 0.5480660200119019, + "score": -0.4435648840971451 + }, + "val": { + "rmse": 0.4627297681263241, + "mae": 0.35958367586135864, + "r2": 0.5089557766914368, + "score": -0.4627297681263241 + }, + "test": { + "rmse": 0.48025858554781287, + "mae": 0.3747011125087738, + "r2": 0.485754132270813, + "score": -0.48025858554781287 + } + }, + "time": "0:23:02.905011", + "diversity": { + "train": { + "mean_centered_corr": -0.020404240116477013, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0513404980301857 + }, + "val": { + "mean_centered_corr": -0.02041091024875641, + "mean_pairwise_disagreement": NaN, + "member_std": 0.052927520126104355 + }, + "test": { + "mean_centered_corr": -0.020323792472481728, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05353504791855812 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.0001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.0001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.0001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.0001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..8c8acd951 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.0001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,177 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44353548828929634, + "mae": 0.3452431261539459, + "r2": 0.5481259822845459, + "score": -0.44353548828929634 + }, + "val": { + "rmse": 0.4624375326912171, + "mae": 0.3594800531864166, + "r2": 0.5095758438110352, + "score": -0.4624375326912171 + }, + "test": { + "rmse": 0.48004674599435365, + "mae": 0.3747219443321228, + "r2": 0.48620766401290894, + "score": -0.48004674599435365 + } + }, + "time": "0:23:02.905011", + "diversity": { + "train": { + "mean_centered_corr": -0.020404240116477013, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0513404980301857 + }, + "val": { + "mean_centered_corr": -0.02041091024875641, + "mean_pairwise_disagreement": NaN, + "member_std": 0.052927520126104355 + }, + "test": { + "mean_centered_corr": -0.020323792472481728, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05353504791855812 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 7, + 24, + 17, + 14, + 5, + 3, + 16, + 4, + 10, + 6, + 22, + 17, + 27, + 24, + 21, + 5, + 14, + 31, + 17, + 25, + 24, + 28 + ], + "head_selection_time": "0:00:00.985957" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..8abb38b47 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.001-best-head-evaluation/0/report.json @@ -0,0 +1,156 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4475775717786638, + "mae": 0.3483501672744751, + "r2": 0.5398522615432739, + "score": -0.4475775717786638 + }, + "val": { + "rmse": 0.4649795846661889, + "mae": 0.361278235912323, + "r2": 0.504169225692749, + "score": -0.4649795846661889 + }, + "test": { + "rmse": 0.4827695666979872, + "mae": 0.3767538070678711, + "r2": 0.4803627133369446, + "score": -0.4827695666979872 + } + }, + "time": "0:23:06.357271", + "diversity": { + "train": { + "mean_centered_corr": -0.02037157490849495, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0513317808508873 + }, + "val": { + "mean_centered_corr": -0.020479049533605576, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05293743312358856 + }, + "test": { + "mean_centered_corr": -0.02046302892267704, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05355745181441307 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 7 + ], + "head_selection_time": "0:00:00.028943" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.001-evaluation/0.toml new file mode 100644 index 000000000..364d0e3c9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.001-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.001-evaluation/0/report.json new file mode 100644 index 000000000..35053ecb2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.001-evaluation/0/report.json @@ -0,0 +1,152 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4435449287553804, + "mae": 0.3451736271381378, + "r2": 0.5481066703796387, + "score": -0.4435449287553804 + }, + "val": { + "rmse": 0.46272482498106277, + "mae": 0.3595674932003021, + "r2": 0.5089662671089172, + "score": -0.46272482498106277 + }, + "test": { + "rmse": 0.4802920784266626, + "mae": 0.37473270297050476, + "r2": 0.4856823682785034, + "score": -0.4802920784266626 + } + }, + "time": "0:23:06.357271", + "diversity": { + "train": { + "mean_centered_corr": -0.02037157490849495, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0513317808508873 + }, + "val": { + "mean_centered_corr": -0.020479049533605576, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05293743312358856 + }, + "test": { + "mean_centered_corr": -0.02046302892267704, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05355745181441307 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..298a7858c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,178 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44362437525125054, + "mae": 0.3453079164028168, + "r2": 0.5479447841644287, + "score": -0.44362437525125054 + }, + "val": { + "rmse": 0.4624107384246987, + "mae": 0.35940805077552795, + "r2": 0.5096326470375061, + "score": -0.4624107384246987 + }, + "test": { + "rmse": 0.479974942646056, + "mae": 0.37463974952697754, + "r2": 0.48636138439178467, + "score": -0.479974942646056 + } + }, + "time": "0:23:06.357271", + "diversity": { + "train": { + "mean_centered_corr": -0.02037157490849495, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0513317808508873 + }, + "val": { + "mean_centered_corr": -0.020479049533605576, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05293743312358856 + }, + "test": { + "mean_centered_corr": -0.02046302892267704, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05355745181441307 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 7, + 31, + 14, + 17, + 3, + 4, + 5, + 24, + 27, + 22, + 10, + 6, + 14, + 31, + 17, + 3, + 5, + 16, + 24, + 4, + 9, + 27, + 14 + ], + "head_selection_time": "0:00:01.063797" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise1e-05-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise1e-05-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise1e-05-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise1e-05-best-head-evaluation/0/report.json new file mode 100644 index 000000000..a9272f18a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise1e-05-best-head-evaluation/0/report.json @@ -0,0 +1,156 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44782485316635157, + "mae": 0.348581999540329, + "r2": 0.5393436551094055, + "score": -0.44782485316635157 + }, + "val": { + "rmse": 0.465206517587419, + "mae": 0.36141353845596313, + "r2": 0.5036851167678833, + "score": -0.465206517587419 + }, + "test": { + "rmse": 0.4827162118838719, + "mae": 0.3766613304615021, + "r2": 0.48047757148742676, + "score": -0.4827162118838719 + } + }, + "time": "0:23:03.312853", + "diversity": { + "train": { + "mean_centered_corr": -0.02040313556790352, + "mean_pairwise_disagreement": NaN, + "member_std": 0.051265861839056015 + }, + "val": { + "mean_centered_corr": -0.02042475715279579, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05283457785844803 + }, + "test": { + "mean_centered_corr": -0.02050917036831379, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05348861590027809 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 7 + ], + "head_selection_time": "0:00:00.027749" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise1e-05-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise1e-05-evaluation/0.toml new file mode 100644 index 000000000..1f9317e7e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise1e-05-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise1e-05-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise1e-05-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise1e-05-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise1e-05-evaluation/0/report.json new file mode 100644 index 000000000..0aa0dbfa1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise1e-05-evaluation/0/report.json @@ -0,0 +1,152 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44357932932311045, + "mae": 0.34517449140548706, + "r2": 0.5480365753173828, + "score": -0.44357932932311045 + }, + "val": { + "rmse": 0.4627757191386438, + "mae": 0.3595951199531555, + "r2": 0.5088582038879395, + "score": -0.4627757191386438 + }, + "test": { + "rmse": 0.4802501150008962, + "mae": 0.37466320395469666, + "r2": 0.4857722520828247, + "score": -0.4802501150008962 + } + }, + "time": "0:23:03.312853", + "diversity": { + "train": { + "mean_centered_corr": -0.02040313556790352, + "mean_pairwise_disagreement": NaN, + "member_std": 0.051265861839056015 + }, + "val": { + "mean_centered_corr": -0.02042475715279579, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05283457785844803 + }, + "test": { + "mean_centered_corr": -0.02050917036831379, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05348861590027809 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise1e-05-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise1e-05-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise1e-05-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise1e-05-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..c4595c43b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise1e-05-greedy-heads-evaluation/0/report.json @@ -0,0 +1,180 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4435210080380358, + "mae": 0.3451911509037018, + "r2": 0.5481554269790649, + "score": -0.4435210080380358 + }, + "val": { + "rmse": 0.46247985576887213, + "mae": 0.35945838689804077, + "r2": 0.5094860792160034, + "score": -0.46247985576887213 + }, + "test": { + "rmse": 0.47992046986293, + "mae": 0.3745458126068115, + "r2": 0.48647797107696533, + "score": -0.47992046986293 + } + }, + "time": "0:23:03.312853", + "diversity": { + "train": { + "mean_centered_corr": -0.02040313556790352, + "mean_pairwise_disagreement": NaN, + "member_std": 0.051265861839056015 + }, + "val": { + "mean_centered_corr": -0.02042475715279579, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05283457785844803 + }, + "test": { + "mean_centered_corr": -0.02050917036831379, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05348861590027809 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 7, + 5, + 22, + 27, + 24, + 14, + 17, + 15, + 10, + 3, + 31, + 5, + 16, + 20, + 25, + 4, + 14, + 27, + 17, + 22, + 5, + 10, + 3, + 21, + 24 + ], + "head_selection_time": "0:00:01.189782" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.0001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.0001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.0001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.0001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..04178817a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.0001-best-head-evaluation/0/report.json @@ -0,0 +1,156 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 2174880, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.447447843788818, + "mae": 0.34833618998527527, + "r2": 0.540118932723999, + "score": -0.447447843788818 + }, + "val": { + "rmse": 0.46500539773966587, + "mae": 0.3613583743572235, + "r2": 0.5041141510009766, + "score": -0.46500539773966587 + }, + "test": { + "rmse": 0.4826475534492275, + "mae": 0.3766460120677948, + "r2": 0.48062533140182495, + "score": -0.4826475534492275 + } + }, + "time": "0:40:10.004190", + "diversity": { + "train": { + "mean_centered_corr": -0.020381338894367218, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05130600184202194 + }, + "val": { + "mean_centered_corr": -0.020470399409532547, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05295143276453018 + }, + "test": { + "mean_centered_corr": -0.020449768751859665, + "mean_pairwise_disagreement": NaN, + "member_std": 0.053552430123090744 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 7 + ], + "head_selection_time": "0:00:00.027926" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.0001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.0001-evaluation/0.toml new file mode 100644 index 000000000..d8590d7b9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.0001-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 8 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.0001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.0001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.0001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.0001-evaluation/0/report.json new file mode 100644 index 000000000..87dcf6175 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.0001-evaluation/0/report.json @@ -0,0 +1,152 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 2174880, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4433250077829461, + "mae": 0.34498268365859985, + "r2": 0.5485547184944153, + "score": -0.4433250077829461 + }, + "val": { + "rmse": 0.4627781501968618, + "mae": 0.3595951795578003, + "r2": 0.5088530778884888, + "score": -0.4627781501968618 + }, + "test": { + "rmse": 0.4803483238337491, + "mae": 0.3747539520263672, + "r2": 0.48556190729141235, + "score": -0.4803483238337491 + } + }, + "time": "0:40:10.004190", + "diversity": { + "train": { + "mean_centered_corr": -0.020381338894367218, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05130600184202194 + }, + "val": { + "mean_centered_corr": -0.020470399409532547, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05295143276453018 + }, + "test": { + "mean_centered_corr": -0.020449768751859665, + "mean_pairwise_disagreement": NaN, + "member_std": 0.053552430123090744 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.0001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.0001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.0001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.0001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..d10aadcfd --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.0001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,176 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 2174880, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4433834556728311, + "mae": 0.34509384632110596, + "r2": 0.5484356880187988, + "score": -0.4433834556728311 + }, + "val": { + "rmse": 0.4625085630139665, + "mae": 0.3594866991043091, + "r2": 0.509425163269043, + "score": -0.4625085630139665 + }, + "test": { + "rmse": 0.4800564927891271, + "mae": 0.3746797442436218, + "r2": 0.48618680238723755, + "score": -0.4800564927891271 + } + }, + "time": "0:40:10.004190", + "diversity": { + "train": { + "mean_centered_corr": -0.020381338894367218, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05130600184202194 + }, + "val": { + "mean_centered_corr": -0.020470399409532547, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05295143276453018 + }, + "test": { + "mean_centered_corr": -0.020449768751859665, + "mean_pairwise_disagreement": NaN, + "member_std": 0.053552430123090744 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 7, + 17, + 3, + 5, + 14, + 10, + 6, + 20, + 24, + 25, + 4, + 31, + 27, + 17, + 3, + 5, + 21, + 10, + 0, + 22, + 14 + ], + "head_selection_time": "0:00:00.965959" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..a42c8b43a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.001-best-head-evaluation/0/report.json @@ -0,0 +1,156 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 2174880, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44626391984830116, + "mae": 0.3475828170776367, + "r2": 0.5425493717193604, + "score": -0.44626391984830116 + }, + "val": { + "rmse": 0.4648571653594056, + "mae": 0.3613499701023102, + "r2": 0.5044302940368652, + "score": -0.4648571653594056 + }, + "test": { + "rmse": 0.4833629710740515, + "mae": 0.37732839584350586, + "r2": 0.47908449172973633, + "score": -0.4833629710740515 + } + }, + "time": "0:40:00.899069", + "diversity": { + "train": { + "mean_centered_corr": -0.020409030839800835, + "mean_pairwise_disagreement": NaN, + "member_std": 0.051384977996349335 + }, + "val": { + "mean_centered_corr": -0.020344730466604233, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05295592173933983 + }, + "test": { + "mean_centered_corr": -0.02032282203435898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.053611382842063904 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 17 + ], + "head_selection_time": "0:00:00.029090" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.001-evaluation/0.toml new file mode 100644 index 000000000..952aefeab --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.001-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 8 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.001-evaluation/0/report.json new file mode 100644 index 000000000..f340087bc --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.001-evaluation/0/report.json @@ -0,0 +1,152 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 2174880, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44330914250619985, + "mae": 0.3449688255786896, + "r2": 0.5485870838165283, + "score": -0.44330914250619985 + }, + "val": { + "rmse": 0.4627485258356125, + "mae": 0.3595917522907257, + "r2": 0.5089159607887268, + "score": -0.4627485258356125 + }, + "test": { + "rmse": 0.4803784913673949, + "mae": 0.37476053833961487, + "r2": 0.48549729585647583, + "score": -0.4803784913673949 + } + }, + "time": "0:40:00.899069", + "diversity": { + "train": { + "mean_centered_corr": -0.020409030839800835, + "mean_pairwise_disagreement": NaN, + "member_std": 0.051384977996349335 + }, + "val": { + "mean_centered_corr": -0.020344730466604233, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05295592173933983 + }, + "test": { + "mean_centered_corr": -0.02032282203435898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.053611382842063904 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..c612469d5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,164 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 2174880, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44316496767007946, + "mae": 0.34489932656288147, + "r2": 0.5488806366920471, + "score": -0.44316496767007946 + }, + "val": { + "rmse": 0.4624392405112558, + "mae": 0.3594130277633667, + "r2": 0.5095721483230591, + "score": -0.4624392405112558 + }, + "test": { + "rmse": 0.4799761534270821, + "mae": 0.3745183050632477, + "r2": 0.48635876178741455, + "score": -0.4799761534270821 + } + }, + "time": "0:40:00.899069", + "diversity": { + "train": { + "mean_centered_corr": -0.020409030839800835, + "mean_pairwise_disagreement": NaN, + "member_std": 0.051384977996349335 + }, + "val": { + "mean_centered_corr": -0.020344730466604233, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05295592173933983 + }, + "test": { + "mean_centered_corr": -0.02032282203435898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.053611382842063904 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 17, + 3, + 5, + 14, + 27, + 24, + 4, + 25, + 22 + ], + "head_selection_time": "0:00:00.314664" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise1e-05-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise1e-05-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise1e-05-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise1e-05-best-head-evaluation/0/report.json new file mode 100644 index 000000000..0a3114588 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise1e-05-best-head-evaluation/0/report.json @@ -0,0 +1,156 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 2174880, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4460647319702049, + "mae": 0.3470715880393982, + "r2": 0.5429576635360718, + "score": -0.4460647319702049 + }, + "val": { + "rmse": 0.46497304704958375, + "mae": 0.36152011156082153, + "r2": 0.5041831731796265, + "score": -0.46497304704958375 + }, + "test": { + "rmse": 0.4824692397862243, + "mae": 0.3767029345035553, + "r2": 0.48100900650024414, + "score": -0.4824692397862243 + } + }, + "time": "0:40:06.616857", + "diversity": { + "train": { + "mean_centered_corr": -0.020358718931674957, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0513102225959301 + }, + "val": { + "mean_centered_corr": -0.020433256402611732, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05285926163196564 + }, + "test": { + "mean_centered_corr": -0.020339958369731903, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05363500863313675 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 14 + ], + "head_selection_time": "0:00:00.028964" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise1e-05-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise1e-05-evaluation/0.toml new file mode 100644 index 000000000..80aaa0c1a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise1e-05-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 8 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise1e-05-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise1e-05-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise1e-05-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise1e-05-evaluation/0/report.json new file mode 100644 index 000000000..dfaebb48b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise1e-05-evaluation/0/report.json @@ -0,0 +1,152 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 2174880, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.443289545416571, + "mae": 0.3449470102787018, + "r2": 0.5486269593238831, + "score": -0.443289545416571 + }, + "val": { + "rmse": 0.46274024999580365, + "mae": 0.3595466911792755, + "r2": 0.5089335441589355, + "score": -0.46274024999580365 + }, + "test": { + "rmse": 0.48030301468765013, + "mae": 0.374723345041275, + "r2": 0.4856589436531067, + "score": -0.48030301468765013 + } + }, + "time": "0:40:06.616857", + "diversity": { + "train": { + "mean_centered_corr": -0.020358718931674957, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0513102225959301 + }, + "val": { + "mean_centered_corr": -0.020433256402611732, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05285926163196564 + }, + "test": { + "mean_centered_corr": -0.020339958369731903, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05363500863313675 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise1e-05-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise1e-05-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise1e-05-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise1e-05-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..dfd22beb3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise1e-05-greedy-heads-evaluation/0/report.json @@ -0,0 +1,170 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 2174880, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4431708014670606, + "mae": 0.34492170810699463, + "r2": 0.548868715763092, + "score": -0.4431708014670606 + }, + "val": { + "rmse": 0.46244718340270563, + "mae": 0.3594217300415039, + "r2": 0.5095553398132324, + "score": -0.46244718340270563 + }, + "test": { + "rmse": 0.4799796305248435, + "mae": 0.37459227442741394, + "r2": 0.4863513112068176, + "score": -0.4799796305248435 + } + }, + "time": "0:40:06.616857", + "diversity": { + "train": { + "mean_centered_corr": -0.020358718931674957, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0513102225959301 + }, + "val": { + "mean_centered_corr": -0.020433256402611732, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05285926163196564 + }, + "test": { + "mean_centered_corr": -0.020339958369731903, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05363500863313675 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 14, + 3, + 5, + 17, + 10, + 27, + 24, + 4, + 20, + 22, + 5, + 21, + 3, + 25, + 6 + ], + "head_selection_time": "0:00:00.612249" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_four_combined-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_four_combined-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_four_combined-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_four_combined-best-head-evaluation/0/report.json new file mode 100644 index 000000000..8043c4861 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_four_combined-best-head-evaluation/0/report.json @@ -0,0 +1,506 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9009943008422852, + "min_features_per_member": 188.0, + "mean_features_per_member": 198.21875, + "max_features_per_member": 206.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.831817626953125, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8196317439934785, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_four_combined-evaluation/_esam_diagnostics/all_four_combined-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5163498609634084, + "mae": 0.40700992941856384, + "r2": 0.500271201133728, + "score": -0.5163498609634084 + }, + "val": { + "rmse": 0.557107642501001, + "mae": 0.44366100430488586, + "r2": 0.4147301912307739, + "score": -0.557107642501001 + }, + "test": { + "rmse": 0.5520572896128711, + "mae": 0.4460996389389038, + "r2": 0.43209290504455566, + "score": -0.5520572896128711 + } + }, + "time": "0:30:06.285335", + "diversity": { + "train": { + "mean_centered_corr": -0.019723476842045784, + "mean_pairwise_disagreement": NaN, + "member_std": 0.059378866106271744 + }, + "val": { + "mean_centered_corr": -0.019546858966350555, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06028679013252258 + }, + "test": { + "mean_centered_corr": -0.0195509921759367, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05971246585249901 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 24 + ], + "head_selection_time": "0:00:00.023809" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_four_combined-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_four_combined-evaluation/0.toml new file mode 100644 index 000000000..48f767b9b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_four_combined-evaluation/0.toml @@ -0,0 +1,74 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 81 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "delivery-eta" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_four_combined-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_four_combined-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_four_combined-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_four_combined-evaluation/0/report.json new file mode 100644 index 000000000..fe2a56876 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_four_combined-evaluation/0/report.json @@ -0,0 +1,502 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9009943008422852, + "min_features_per_member": 188.0, + "mean_features_per_member": 198.21875, + "max_features_per_member": 206.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.831817626953125, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8196317439934785, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_four_combined-evaluation/_esam_diagnostics/all_four_combined-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5114855052725168, + "mae": 0.4018738567829132, + "r2": 0.5096423625946045, + "score": -0.5114855052725168 + }, + "val": { + "rmse": 0.5560789252392828, + "mae": 0.44038042426109314, + "r2": 0.41688960790634155, + "score": -0.5560789252392828 + }, + "test": { + "rmse": 0.5493995976501785, + "mae": 0.44166451692581177, + "r2": 0.4375477433204651, + "score": -0.5493995976501785 + } + }, + "time": "0:30:06.285335", + "diversity": { + "train": { + "mean_centered_corr": -0.019723476842045784, + "mean_pairwise_disagreement": NaN, + "member_std": 0.059378866106271744 + }, + "val": { + "mean_centered_corr": -0.019546858966350555, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06028679013252258 + }, + "test": { + "mean_centered_corr": -0.0195509921759367, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05971246585249901 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_four_combined-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_four_combined-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_four_combined-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_four_combined-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..d27b42061 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_four_combined-greedy-heads-evaluation/0/report.json @@ -0,0 +1,510 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9009943008422852, + "min_features_per_member": 188.0, + "mean_features_per_member": 198.21875, + "max_features_per_member": 206.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.831817626953125, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8196317439934785, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_four_combined-evaluation/_esam_diagnostics/all_four_combined-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5123158821979983, + "mae": 0.40332356095314026, + "r2": 0.5080488920211792, + "score": -0.5123158821979983 + }, + "val": { + "rmse": 0.5558210261849236, + "mae": 0.4417085349559784, + "r2": 0.41743040084838867, + "score": -0.5558210261849236 + }, + "test": { + "rmse": 0.551614874495521, + "mae": 0.4454870820045471, + "r2": 0.433002769947052, + "score": -0.551614874495521 + } + }, + "time": "0:30:06.285335", + "diversity": { + "train": { + "mean_centered_corr": -0.019723476842045784, + "mean_pairwise_disagreement": NaN, + "member_std": 0.059378866106271744 + }, + "val": { + "mean_centered_corr": -0.019546858966350555, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06028679013252258 + }, + "test": { + "mean_centered_corr": -0.0195509921759367, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05971246585249901 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 24, + 30, + 17, + 8, + 12 + ], + "head_selection_time": "0:00:00.148062" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_cf_fisd-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_cf_fisd-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_cf_fisd-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_cf_fisd-best-head-evaluation/0/report.json new file mode 100644 index 000000000..71c39dbe8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_cf_fisd-best-head-evaluation/0/report.json @@ -0,0 +1,225 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9009943008422852, + "min_features_per_member": 188.0, + "mean_features_per_member": 198.21875, + "max_features_per_member": 206.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.831817626953125, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8196317439934785, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_cf_fisd-evaluation/_esam_diagnostics/all_minus_cf_fisd-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5125628200269988, + "mae": 0.40779775381088257, + "r2": 0.5075745582580566, + "score": -0.5125628200269988 + }, + "val": { + "rmse": 0.5589936165675472, + "mae": 0.4451264441013336, + "r2": 0.41076087951660156, + "score": -0.5589936165675472 + }, + "test": { + "rmse": 0.5554125439499674, + "mae": 0.44996029138565063, + "r2": 0.4251687526702881, + "score": -0.5554125439499674 + } + }, + "time": "0:31:01.859750", + "diversity": { + "train": { + "mean_centered_corr": -0.020016251131892204, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07126544415950775 + }, + "val": { + "mean_centered_corr": -0.020217863842844963, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0740198865532875 + }, + "test": { + "mean_centered_corr": -0.020295117050409317, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0730525478720665 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023553" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_cf_fisd-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_cf_fisd-evaluation/0.toml new file mode 100644 index 000000000..0747ce77a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_cf_fisd-evaluation/0.toml @@ -0,0 +1,63 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_cf_fisd-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_cf_fisd-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_cf_fisd-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_cf_fisd-evaluation/0/report.json new file mode 100644 index 000000000..70c1917f9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_cf_fisd-evaluation/0/report.json @@ -0,0 +1,221 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9009943008422852, + "min_features_per_member": 188.0, + "mean_features_per_member": 198.21875, + "max_features_per_member": 206.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.831817626953125, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8196317439934785, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_cf_fisd-evaluation/_esam_diagnostics/all_minus_cf_fisd-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5021333185484521, + "mae": 0.39498522877693176, + "r2": 0.5274102091789246, + "score": -0.5021333185484521 + }, + "val": { + "rmse": 0.5563682291205534, + "mae": 0.44030144810676575, + "r2": 0.4162827730178833, + "score": -0.5563682291205534 + }, + "test": { + "rmse": 0.5506428213190716, + "mae": 0.44258415699005127, + "r2": 0.43499934673309326, + "score": -0.5506428213190716 + } + }, + "time": "0:31:01.859750", + "diversity": { + "train": { + "mean_centered_corr": -0.020016251131892204, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07126544415950775 + }, + "val": { + "mean_centered_corr": -0.020217863842844963, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0740198865532875 + }, + "test": { + "mean_centered_corr": -0.020295117050409317, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0730525478720665 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_cf_fisd-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_cf_fisd-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_cf_fisd-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_cf_fisd-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..96f7f509c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_cf_fisd-greedy-heads-evaluation/0/report.json @@ -0,0 +1,236 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9009943008422852, + "min_features_per_member": 188.0, + "mean_features_per_member": 198.21875, + "max_features_per_member": 206.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.831817626953125, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8196317439934785, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_cf_fisd-evaluation/_esam_diagnostics/all_minus_cf_fisd-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5032708353541949, + "mae": 0.39703547954559326, + "r2": 0.5252666473388672, + "score": -0.5032708353541949 + }, + "val": { + "rmse": 0.5557239681192615, + "mae": 0.44123438000679016, + "r2": 0.4176338315010071, + "score": -0.5557239681192615 + }, + "test": { + "rmse": 0.5525121073453494, + "mae": 0.44600483775138855, + "r2": 0.43115681409835815, + "score": -0.5525121073453494 + } + }, + "time": "0:31:01.859750", + "diversity": { + "train": { + "mean_centered_corr": -0.020016251131892204, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07126544415950775 + }, + "val": { + "mean_centered_corr": -0.020217863842844963, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0740198865532875 + }, + "test": { + "mean_centered_corr": -0.020295117050409317, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0730525478720665 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 13, + 14, + 15, + 17, + 21, + 2, + 29, + 12, + 14, + 22, + 4, + 23 + ], + "head_selection_time": "0:00:00.359604" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_esam-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_esam-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_esam-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_esam-best-head-evaluation/0/report.json new file mode 100644 index 000000000..ca02d9cd6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_esam-best-head-evaluation/0/report.json @@ -0,0 +1,496 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9009943008422852, + "min_features_per_member": 188.0, + "mean_features_per_member": 198.21875, + "max_features_per_member": 206.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.831817626953125, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8196317439934785, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5186929177280425, + "mae": 0.40856218338012695, + "r2": 0.4957256317138672, + "score": -0.5186929177280425 + }, + "val": { + "rmse": 0.5574872455078781, + "mae": 0.44453755021095276, + "r2": 0.41393232345581055, + "score": -0.5574872455078781 + }, + "test": { + "rmse": 0.5557374285650164, + "mae": 0.4497009515762329, + "r2": 0.4244961142539978, + "score": -0.5557374285650164 + } + }, + "time": "0:16:51.220052", + "diversity": { + "train": { + "mean_centered_corr": -0.019181909039616585, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05811057612299919 + }, + "val": { + "mean_centered_corr": -0.01869083382189274, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05933859571814537 + }, + "test": { + "mean_centered_corr": -0.01882953569293022, + "mean_pairwise_disagreement": NaN, + "member_std": 0.058781228959560394 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 24 + ], + "head_selection_time": "0:00:00.023324" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_esam-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_esam-evaluation/0.toml new file mode 100644 index 000000000..6c47aea17 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_esam-evaluation/0.toml @@ -0,0 +1,64 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 81 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "delivery-eta" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_esam-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_esam-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_esam-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_esam-evaluation/0/report.json new file mode 100644 index 000000000..99f6ba59c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_esam-evaluation/0/report.json @@ -0,0 +1,492 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9009943008422852, + "min_features_per_member": 188.0, + "mean_features_per_member": 198.21875, + "max_features_per_member": 206.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.831817626953125, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8196317439934785, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5143036239160346, + "mae": 0.4036540389060974, + "r2": 0.5042240619659424, + "score": -0.5143036239160346 + }, + "val": { + "rmse": 0.556102345189303, + "mae": 0.44109949469566345, + "r2": 0.41684049367904663, + "score": -0.556102345189303 + }, + "test": { + "rmse": 0.5505710768952337, + "mae": 0.4436887204647064, + "r2": 0.4351465702056885, + "score": -0.5505710768952337 + } + }, + "time": "0:16:51.220052", + "diversity": { + "train": { + "mean_centered_corr": -0.019181909039616585, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05811057612299919 + }, + "val": { + "mean_centered_corr": -0.01869083382189274, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05933859571814537 + }, + "test": { + "mean_centered_corr": -0.01882953569293022, + "mean_pairwise_disagreement": NaN, + "member_std": 0.058781228959560394 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_esam-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_esam-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_esam-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_esam-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..0af0654bc --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_esam-greedy-heads-evaluation/0/report.json @@ -0,0 +1,502 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9009943008422852, + "min_features_per_member": 188.0, + "mean_features_per_member": 198.21875, + "max_features_per_member": 206.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.831817626953125, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8196317439934785, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5147739959374277, + "mae": 0.40450477600097656, + "r2": 0.5033168196678162, + "score": -0.5147739959374277 + }, + "val": { + "rmse": 0.5556806886998003, + "mae": 0.4414132237434387, + "r2": 0.41772449016571045, + "score": -0.5556806886998003 + }, + "test": { + "rmse": 0.5513624006625033, + "mae": 0.4449708163738251, + "r2": 0.43352168798446655, + "score": -0.5513624006625033 + } + }, + "time": "0:16:51.220052", + "diversity": { + "train": { + "mean_centered_corr": -0.019181909039616585, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05811057612299919 + }, + "val": { + "mean_centered_corr": -0.01869083382189274, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05933859571814537 + }, + "test": { + "mean_centered_corr": -0.01882953569293022, + "mean_pairwise_disagreement": NaN, + "member_std": 0.058781228959560394 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 24, + 0, + 18, + 19, + 30, + 6, + 28 + ], + "head_selection_time": "0:00:00.202098" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_mfb-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_mfb-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_mfb-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_mfb-best-head-evaluation/0/report.json new file mode 100644 index 000000000..6262c0e23 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_mfb-best-head-evaluation/0/report.json @@ -0,0 +1,470 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_mfb-evaluation/_esam_diagnostics/all_minus_mfb-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5099830602188714, + "mae": 0.40115422010421753, + "r2": 0.5125188827514648, + "score": -0.5099830602188714 + }, + "val": { + "rmse": 0.5591485795972553, + "mae": 0.4450674057006836, + "r2": 0.41043412685394287, + "score": -0.5591485795972553 + }, + "test": { + "rmse": 0.5570234889363207, + "mae": 0.45008188486099243, + "r2": 0.4218294024467468, + "score": -0.5570234889363207 + } + }, + "time": "0:30:38.784268", + "diversity": { + "train": { + "mean_centered_corr": -0.019464364275336266, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06317681819200516 + }, + "val": { + "mean_centered_corr": -0.019761033356189728, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06557753682136536 + }, + "test": { + "mean_centered_corr": -0.019694572314620018, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06528102606534958 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 29 + ], + "head_selection_time": "0:00:00.023362" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_mfb-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_mfb-evaluation/0.toml new file mode 100644 index 000000000..79844057e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_mfb-evaluation/0.toml @@ -0,0 +1,61 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "delivery-eta" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_mfb-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_mfb-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_mfb-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_mfb-evaluation/0/report.json new file mode 100644 index 000000000..74ce93d9d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_mfb-evaluation/0/report.json @@ -0,0 +1,466 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_mfb-evaluation/_esam_diagnostics/all_minus_mfb-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5030298522163873, + "mae": 0.3948743939399719, + "r2": 0.5257211327552795, + "score": -0.5030298522163873 + }, + "val": { + "rmse": 0.5571737580629273, + "mae": 0.4427817463874817, + "r2": 0.4145912528038025, + "score": -0.5571737580629273 + }, + "test": { + "rmse": 0.5527981832493051, + "mae": 0.445404052734375, + "r2": 0.43056756258010864, + "score": -0.5527981832493051 + } + }, + "time": "0:30:38.784268", + "diversity": { + "train": { + "mean_centered_corr": -0.019464364275336266, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06317681819200516 + }, + "val": { + "mean_centered_corr": -0.019761033356189728, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06557753682136536 + }, + "test": { + "mean_centered_corr": -0.019694572314620018, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06528102606534958 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_mfb-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_mfb-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_mfb-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_mfb-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..8d0a2e3cf --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_mfb-greedy-heads-evaluation/0/report.json @@ -0,0 +1,475 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_mfb-evaluation/_esam_diagnostics/all_minus_mfb-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5034348700541703, + "mae": 0.39474746584892273, + "r2": 0.5249570608139038, + "score": -0.5034348700541703 + }, + "val": { + "rmse": 0.5566172544693042, + "mae": 0.44152215123176575, + "r2": 0.4157600998878479, + "score": -0.5566172544693042 + }, + "test": { + "rmse": 0.5524610510858833, + "mae": 0.4448763132095337, + "r2": 0.43126189708709717, + "score": -0.5524610510858833 + } + }, + "time": "0:30:38.784268", + "diversity": { + "train": { + "mean_centered_corr": -0.019464364275336266, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06317681819200516 + }, + "val": { + "mean_centered_corr": -0.019761033356189728, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06557753682136536 + }, + "test": { + "mean_centered_corr": -0.019694572314620018, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06528102606534958 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 29, + 10, + 13, + 15, + 24, + 16 + ], + "head_selection_time": "0:00:00.172357" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_rla-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_rla-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_rla-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_rla-best-head-evaluation/0/report.json new file mode 100644 index 000000000..a9d3e5d77 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_rla-best-head-evaluation/0/report.json @@ -0,0 +1,495 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9009943008422852, + "min_features_per_member": 188.0, + "mean_features_per_member": 198.21875, + "max_features_per_member": 206.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.831817626953125, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8196317439934785, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_rla-evaluation/_esam_diagnostics/all_minus_rla-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5181245330790036, + "mae": 0.4071075916290283, + "r2": 0.49683016538619995, + "score": -0.5181245330790036 + }, + "val": { + "rmse": 0.5572234465535005, + "mae": 0.44068703055381775, + "r2": 0.414486825466156, + "score": -0.5572234465535005 + }, + "test": { + "rmse": 0.5525491087220346, + "mae": 0.44485196471214294, + "r2": 0.43108057975769043, + "score": -0.5525491087220346 + } + }, + "time": "0:14:53.713644", + "diversity": { + "train": { + "mean_centered_corr": -0.018655147403478622, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06323739886283875 + }, + "val": { + "mean_centered_corr": -0.018664149567484856, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06284364312887192 + }, + "test": { + "mean_centered_corr": -0.018706250935792923, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06153913214802742 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 25 + ], + "head_selection_time": "0:00:00.023432" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_rla-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_rla-evaluation/0.toml new file mode 100644 index 000000000..023666cec --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_rla-evaluation/0.toml @@ -0,0 +1,67 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 81 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "delivery-eta" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_rla-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_rla-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_rla-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_rla-evaluation/0/report.json new file mode 100644 index 000000000..3cf3fcc90 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_rla-evaluation/0/report.json @@ -0,0 +1,491 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9009943008422852, + "min_features_per_member": 188.0, + "mean_features_per_member": 198.21875, + "max_features_per_member": 206.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.831817626953125, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8196317439934785, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_rla-evaluation/_esam_diagnostics/all_minus_rla-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.514037346731799, + "mae": 0.4048083424568176, + "r2": 0.5047372579574585, + "score": -0.514037346731799 + }, + "val": { + "rmse": 0.5560596579733861, + "mae": 0.44129467010498047, + "r2": 0.41693001985549927, + "score": -0.5560596579733861 + }, + "test": { + "rmse": 0.5513667248174312, + "mae": 0.4448263645172119, + "r2": 0.4335128664970398, + "score": -0.5513667248174312 + } + }, + "time": "0:14:53.713644", + "diversity": { + "train": { + "mean_centered_corr": -0.018655147403478622, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06323739886283875 + }, + "val": { + "mean_centered_corr": -0.018664149567484856, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06284364312887192 + }, + "test": { + "mean_centered_corr": -0.018706250935792923, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06153913214802742 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_rla-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_rla-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_rla-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_rla-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..9bc611603 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_rla-greedy-heads-evaluation/0/report.json @@ -0,0 +1,508 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9009943008422852, + "min_features_per_member": 188.0, + "mean_features_per_member": 198.21875, + "max_features_per_member": 206.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.831817626953125, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8196317439934785, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_rla-evaluation/_esam_diagnostics/all_minus_rla-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5148022474310279, + "mae": 0.4057745337486267, + "r2": 0.5032622814178467, + "score": -0.5148022474310279 + }, + "val": { + "rmse": 0.5556365745463094, + "mae": 0.44140082597732544, + "r2": 0.41781699657440186, + "score": -0.5556365745463094 + }, + "test": { + "rmse": 0.5515955863733182, + "mae": 0.44562408328056335, + "r2": 0.43304240703582764, + "score": -0.5515955863733182 + } + }, + "time": "0:14:53.713644", + "diversity": { + "train": { + "mean_centered_corr": -0.018655147403478622, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06323739886283875 + }, + "val": { + "mean_centered_corr": -0.018664149567484856, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06284364312887192 + }, + "test": { + "mean_centered_corr": -0.018706250935792923, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06153913214802742 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 25, + 24, + 30, + 22, + 19, + 29, + 14, + 23, + 31, + 24, + 12, + 30, + 22, + 6 + ], + "head_selection_time": "0:00:00.434304" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.05-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.05-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.05-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.05-best-head-evaluation/0/report.json new file mode 100644 index 000000000..0927cc2e8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.05-best-head-evaluation/0/report.json @@ -0,0 +1,449 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5140202432442316, + "mae": 0.40386128425598145, + "r2": 0.5047702789306641, + "score": -0.5140202432442316 + }, + "val": { + "rmse": 0.557661572508718, + "mae": 0.44313740730285645, + "r2": 0.4135657548904419, + "score": -0.557661572508718 + }, + "test": { + "rmse": 0.553144673358601, + "mae": 0.44622060656547546, + "r2": 0.42985349893569946, + "score": -0.553144673358601 + } + }, + "time": "0:08:00.261340", + "diversity": { + "train": { + "mean_centered_corr": -0.01827983371913433, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0643538236618042 + }, + "val": { + "mean_centered_corr": -0.018152762204408646, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06669643521308899 + }, + "test": { + "mean_centered_corr": -0.018407996743917465, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06605677306652069 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 30 + ], + "head_selection_time": "0:00:00.023441" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.05-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.05-evaluation/0.toml new file mode 100644 index 000000000..f7bf33592 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.05-evaluation/0.toml @@ -0,0 +1,44 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 + +[cf_fisd] +lambda = 0.05 +variant = "raw" +dataset_name = "delivery-eta" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.05-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.05-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.05-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.05-evaluation/0/report.json new file mode 100644 index 000000000..8dc921357 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.05-evaluation/0/report.json @@ -0,0 +1,445 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5080552327962835, + "mae": 0.3986222445964813, + "r2": 0.5161974430084229, + "score": -0.5080552327962835 + }, + "val": { + "rmse": 0.5562445854609275, + "mae": 0.4416752755641937, + "r2": 0.4165421724319458, + "score": -0.5562445854609275 + }, + "test": { + "rmse": 0.5504166498793033, + "mae": 0.44359084963798523, + "r2": 0.4354633688926697, + "score": -0.5504166498793033 + } + }, + "time": "0:08:00.261340", + "diversity": { + "train": { + "mean_centered_corr": -0.01827983371913433, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0643538236618042 + }, + "val": { + "mean_centered_corr": -0.018152762204408646, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06669643521308899 + }, + "test": { + "mean_centered_corr": -0.018407996743917465, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06605677306652069 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.05-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.05-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.05-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.05-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..e627e5168 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.05-greedy-heads-evaluation/0/report.json @@ -0,0 +1,457 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5096923366202544, + "mae": 0.4003257751464844, + "r2": 0.5130745768547058, + "score": -0.5096923366202544 + }, + "val": { + "rmse": 0.5556691040453957, + "mae": 0.4418928027153015, + "r2": 0.4177488088607788, + "score": -0.5556691040453957 + }, + "test": { + "rmse": 0.5521822756750591, + "mae": 0.44639116525650024, + "r2": 0.43183577060699463, + "score": -0.5521822756750591 + } + }, + "time": "0:08:00.261340", + "diversity": { + "train": { + "mean_centered_corr": -0.01827983371913433, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0643538236618042 + }, + "val": { + "mean_centered_corr": -0.018152762204408646, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06669643521308899 + }, + "test": { + "mean_centered_corr": -0.018407996743917465, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06605677306652069 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 30, + 24, + 27, + 15, + 4, + 21, + 24, + 1, + 27 + ], + "head_selection_time": "0:00:00.261765" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.1-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.1-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.1-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.1-best-head-evaluation/0/report.json new file mode 100644 index 000000000..f2a4ae64c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.1-best-head-evaluation/0/report.json @@ -0,0 +1,449 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 3003, + "metrics": { + "train": { + "rmse": 0.5091186451729569, + "mae": 0.40107491612434387, + "r2": 0.5141700506210327, + "score": -0.5091186451729569 + }, + "val": { + "rmse": 0.5580564462352832, + "mae": 0.4435662627220154, + "r2": 0.4127349853515625, + "score": -0.5580564462352832 + }, + "test": { + "rmse": 0.5528824139554394, + "mae": 0.4462261497974396, + "r2": 0.4303940534591675, + "score": -0.5528824139554394 + } + }, + "time": "0:09:02.534268", + "diversity": { + "train": { + "mean_centered_corr": -0.017171135172247887, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07136442512273788 + }, + "val": { + "mean_centered_corr": -0.01788639836013317, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07444944232702255 + }, + "test": { + "mean_centered_corr": -0.017493220046162605, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07251272350549698 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 27 + ], + "head_selection_time": "0:00:00.023820" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.1-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.1-evaluation/0.toml new file mode 100644 index 000000000..26f1234c9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.1-evaluation/0.toml @@ -0,0 +1,44 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "delivery-eta" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.1-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.1-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.1-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.1-evaluation/0/report.json new file mode 100644 index 000000000..8cb92144e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.1-evaluation/0/report.json @@ -0,0 +1,445 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 3003, + "metrics": { + "train": { + "rmse": 0.5000776588212498, + "mae": 0.39380842447280884, + "r2": 0.5312716960906982, + "score": -0.5000776588212498 + }, + "val": { + "rmse": 0.5562691235201772, + "mae": 0.44062381982803345, + "r2": 0.41649067401885986, + "score": -0.5562691235201772 + }, + "test": { + "rmse": 0.5515996385557942, + "mae": 0.4439227879047394, + "r2": 0.4330340623855591, + "score": -0.5515996385557942 + } + }, + "time": "0:09:02.534268", + "diversity": { + "train": { + "mean_centered_corr": -0.017171135172247887, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07136442512273788 + }, + "val": { + "mean_centered_corr": -0.01788639836013317, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07444944232702255 + }, + "test": { + "mean_centered_corr": -0.017493220046162605, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07251272350549698 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.1-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.1-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.1-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.1-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..1b763f269 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.1-greedy-heads-evaluation/0/report.json @@ -0,0 +1,469 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 3003, + "metrics": { + "train": { + "rmse": 0.5022562202858329, + "mae": 0.39594367146492004, + "r2": 0.5271788239479065, + "score": -0.5022562202858329 + }, + "val": { + "rmse": 0.5559326756199947, + "mae": 0.4413962960243225, + "r2": 0.41719627380371094, + "score": -0.5559326756199947 + }, + "test": { + "rmse": 0.5522105562224877, + "mae": 0.4452051818370819, + "r2": 0.4317775368690491, + "score": -0.5522105562224877 + } + }, + "time": "0:09:02.534268", + "diversity": { + "train": { + "mean_centered_corr": -0.017171135172247887, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07136442512273788 + }, + "val": { + "mean_centered_corr": -0.01788639836013317, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07444944232702255 + }, + "test": { + "mean_centered_corr": -0.017493220046162605, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07251272350549698 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 27, + 10, + 7, + 26, + 25, + 17, + 1, + 11, + 28, + 30, + 20, + 9, + 26, + 23, + 19, + 3, + 10, + 12, + 29, + 26, + 24 + ], + "head_selection_time": "0:00:00.749795" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.2-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.2-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.2-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.2-best-head-evaluation/0/report.json new file mode 100644 index 000000000..4df5ac783 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.2-best-head-evaluation/0/report.json @@ -0,0 +1,449 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.2, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.2, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5165139251966278, + "mae": 0.4059557616710663, + "r2": 0.49995356798171997, + "score": -0.5165139251966278 + }, + "val": { + "rmse": 0.5568247172798395, + "mae": 0.4428383708000183, + "r2": 0.41532450914382935, + "score": -0.5568247172798395 + }, + "test": { + "rmse": 0.5526535729970853, + "mae": 0.4464971721172333, + "r2": 0.43086546659469604, + "score": -0.5526535729970853 + } + }, + "time": "0:07:25.928143", + "diversity": { + "train": { + "mean_centered_corr": -0.018287064507603645, + "mean_pairwise_disagreement": NaN, + "member_std": 0.058509234338998795 + }, + "val": { + "mean_centered_corr": -0.01799921505153179, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05907076969742775 + }, + "test": { + "mean_centered_corr": -0.018038002774119377, + "mean_pairwise_disagreement": NaN, + "member_std": 0.057839713990688324 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 25 + ], + "head_selection_time": "0:00:00.023520" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.2-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.2-evaluation/0.toml new file mode 100644 index 000000000..caf96fb13 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.2-evaluation/0.toml @@ -0,0 +1,44 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 + +[cf_fisd] +lambda = 0.2 +variant = "raw" +dataset_name = "delivery-eta" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.2-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.2-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.2-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.2-evaluation/0/report.json new file mode 100644 index 000000000..6f6b85f0e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.2-evaluation/0/report.json @@ -0,0 +1,445 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.2, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.2, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.512394931536762, + "mae": 0.40040844678878784, + "r2": 0.507897138595581, + "score": -0.512394931536762 + }, + "val": { + "rmse": 0.5565664676993906, + "mae": 0.43947651982307434, + "r2": 0.4158666729927063, + "score": -0.5565664676993906 + }, + "test": { + "rmse": 0.5475279860550825, + "mae": 0.438804566860199, + "r2": 0.44137340784072876, + "score": -0.5475279860550825 + } + }, + "time": "0:07:25.928143", + "diversity": { + "train": { + "mean_centered_corr": -0.018287064507603645, + "mean_pairwise_disagreement": NaN, + "member_std": 0.058509234338998795 + }, + "val": { + "mean_centered_corr": -0.01799921505153179, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05907076969742775 + }, + "test": { + "mean_centered_corr": -0.018038002774119377, + "mean_pairwise_disagreement": NaN, + "member_std": 0.057839713990688324 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.2-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.2-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.2-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.2-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..7c53cfab1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.2-greedy-heads-evaluation/0/report.json @@ -0,0 +1,455 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.2, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.2, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5124534399701177, + "mae": 0.40273600816726685, + "r2": 0.5077847242355347, + "score": -0.5124534399701177 + }, + "val": { + "rmse": 0.5558508909198594, + "mae": 0.4416658878326416, + "r2": 0.41736775636672974, + "score": -0.5558508909198594 + }, + "test": { + "rmse": 0.5511616144232316, + "mae": 0.4449461102485657, + "r2": 0.43393421173095703, + "score": -0.5511616144232316 + } + }, + "time": "0:07:25.928143", + "diversity": { + "train": { + "mean_centered_corr": -0.018287064507603645, + "mean_pairwise_disagreement": NaN, + "member_std": 0.058509234338998795 + }, + "val": { + "mean_centered_corr": -0.01799921505153179, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05907076969742775 + }, + "test": { + "mean_centered_corr": -0.018038002774119377, + "mean_pairwise_disagreement": NaN, + "member_std": 0.057839713990688324 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 25, + 31, + 11, + 17, + 8, + 19, + 23 + ], + "head_selection_time": "0:00:00.201039" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_cf_fisd-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_cf_fisd-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_cf_fisd-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_cf_fisd-best-head-evaluation/0/report.json new file mode 100644 index 000000000..263f15f32 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_cf_fisd-best-head-evaluation/0/report.json @@ -0,0 +1,459 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_cf_fisd-evaluation/_esam_diagnostics/esam_cf_fisd-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5178361933092778, + "mae": 0.40899282693862915, + "r2": 0.4973900318145752, + "score": -0.5178361933092778 + }, + "val": { + "rmse": 0.5568538056781229, + "mae": 0.4425808787345886, + "r2": 0.4152633547782898, + "score": -0.5568538056781229 + }, + "test": { + "rmse": 0.5520497317854985, + "mae": 0.4457451403141022, + "r2": 0.43210846185684204, + "score": -0.5520497317854985 + } + }, + "time": "0:13:13.542286", + "diversity": { + "train": { + "mean_centered_corr": -0.018485957756638527, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05812399089336395 + }, + "val": { + "mean_centered_corr": -0.01856246404349804, + "mean_pairwise_disagreement": NaN, + "member_std": 0.059649109840393066 + }, + "test": { + "mean_centered_corr": -0.018637534230947495, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05802324041724205 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 25 + ], + "head_selection_time": "0:00:00.023427" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_cf_fisd-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_cf_fisd-evaluation/0.toml new file mode 100644 index 000000000..82ab734b0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_cf_fisd-evaluation/0.toml @@ -0,0 +1,54 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "delivery-eta" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_cf_fisd-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_cf_fisd-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_cf_fisd-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_cf_fisd-evaluation/0/report.json new file mode 100644 index 000000000..bff979d13 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_cf_fisd-evaluation/0/report.json @@ -0,0 +1,455 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_cf_fisd-evaluation/_esam_diagnostics/esam_cf_fisd-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5138792192419979, + "mae": 0.4050156772136688, + "r2": 0.5050419569015503, + "score": -0.5138792192419979 + }, + "val": { + "rmse": 0.5562165100217151, + "mae": 0.44210323691368103, + "r2": 0.4166010618209839, + "score": -0.5562165100217151 + }, + "test": { + "rmse": 0.5500920359593422, + "mae": 0.4439537823200226, + "r2": 0.436129093170166, + "score": -0.5500920359593422 + } + }, + "time": "0:13:13.542286", + "diversity": { + "train": { + "mean_centered_corr": -0.018485957756638527, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05812399089336395 + }, + "val": { + "mean_centered_corr": -0.01856246404349804, + "mean_pairwise_disagreement": NaN, + "member_std": 0.059649109840393066 + }, + "test": { + "mean_centered_corr": -0.018637534230947495, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05802324041724205 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_cf_fisd-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_cf_fisd-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_cf_fisd-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_cf_fisd-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..13d3b92d6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_cf_fisd-greedy-heads-evaluation/0/report.json @@ -0,0 +1,469 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_cf_fisd-evaluation/_esam_diagnostics/esam_cf_fisd-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.51442579053414, + "mae": 0.40507030487060547, + "r2": 0.50398850440979, + "score": -0.51442579053414 + }, + "val": { + "rmse": 0.5558497381815869, + "mae": 0.4416966438293457, + "r2": 0.4173702001571655, + "score": -0.5558497381815869 + }, + "test": { + "rmse": 0.5508600814353861, + "mae": 0.4445980489253998, + "r2": 0.43455344438552856, + "score": -0.5508600814353861 + } + }, + "time": "0:13:13.542286", + "diversity": { + "train": { + "mean_centered_corr": -0.018485957756638527, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05812399089336395 + }, + "val": { + "mean_centered_corr": -0.01856246404349804, + "mean_pairwise_disagreement": NaN, + "member_std": 0.059649109840393066 + }, + "test": { + "mean_centered_corr": -0.018637534230947495, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05802324041724205 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 25, + 31, + 30, + 21, + 26, + 29, + 24, + 12, + 8, + 31, + 18 + ], + "head_selection_time": "0:00:00.326097" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_mfb-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_mfb-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_mfb-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_mfb-best-head-evaluation/0/report.json new file mode 100644 index 000000000..50d13cf12 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_mfb-best-head-evaluation/0/report.json @@ -0,0 +1,214 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9009943008422852, + "min_features_per_member": 188.0, + "mean_features_per_member": 198.21875, + "max_features_per_member": 206.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.831817626953125, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8196317439934785, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_mfb-evaluation/_esam_diagnostics/esam_mfb-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2730, + "metrics": { + "train": { + "rmse": 0.5044531109629536, + "mae": 0.3947329819202423, + "r2": 0.5230334997177124, + "score": -0.5044531109629536 + }, + "val": { + "rmse": 0.5602888251013873, + "mae": 0.4431866407394409, + "r2": 0.40802711248397827, + "score": -0.5602888251013873 + }, + "test": { + "rmse": 0.5556835311938418, + "mae": 0.44737398624420166, + "r2": 0.42460769414901733, + "score": -0.5556835311938418 + } + }, + "time": "0:15:49.877177", + "diversity": { + "train": { + "mean_centered_corr": -0.020127171650528908, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07589761912822723 + }, + "val": { + "mean_centered_corr": -0.02017100341618061, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07972607016563416 + }, + "test": { + "mean_centered_corr": -0.0202227383852005, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07740911841392517 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 7 + ], + "head_selection_time": "0:00:00.023297" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_mfb-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_mfb-evaluation/0.toml new file mode 100644 index 000000000..1cbf804d8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_mfb-evaluation/0.toml @@ -0,0 +1,56 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_mfb-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_mfb-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_mfb-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_mfb-evaluation/0/report.json new file mode 100644 index 000000000..959f64c32 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_mfb-evaluation/0/report.json @@ -0,0 +1,210 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9009943008422852, + "min_features_per_member": 188.0, + "mean_features_per_member": 198.21875, + "max_features_per_member": 206.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.831817626953125, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8196317439934785, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_mfb-evaluation/_esam_diagnostics/esam_mfb-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2730, + "metrics": { + "train": { + "rmse": 0.4988528443427359, + "mae": 0.39271265268325806, + "r2": 0.5335649251937866, + "score": -0.4988528443427359 + }, + "val": { + "rmse": 0.5560703233852616, + "mae": 0.44045791029930115, + "r2": 0.4169076681137085, + "score": -0.5560703233852616 + }, + "test": { + "rmse": 0.5508616503748457, + "mae": 0.443840891122818, + "r2": 0.4345501661300659, + "score": -0.5508616503748457 + } + }, + "time": "0:15:49.877177", + "diversity": { + "train": { + "mean_centered_corr": -0.020127171650528908, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07589761912822723 + }, + "val": { + "mean_centered_corr": -0.02017100341618061, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07972607016563416 + }, + "test": { + "mean_centered_corr": -0.0202227383852005, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07740911841392517 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_mfb-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_mfb-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_mfb-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_mfb-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..0c954c5a2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_mfb-greedy-heads-evaluation/0/report.json @@ -0,0 +1,233 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9009943008422852, + "min_features_per_member": 188.0, + "mean_features_per_member": 198.21875, + "max_features_per_member": 206.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.831817626953125, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8196317439934785, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_mfb-evaluation/_esam_diagnostics/esam_mfb-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2730, + "metrics": { + "train": { + "rmse": 0.49913719358770847, + "mae": 0.3932115137577057, + "r2": 0.5330330729484558, + "score": -0.49913719358770847 + }, + "val": { + "rmse": 0.5556538720026133, + "mae": 0.4407852590084076, + "r2": 0.41778069734573364, + "score": -0.5556538720026133 + }, + "test": { + "rmse": 0.5520326723090689, + "mae": 0.4453306496143341, + "r2": 0.43214356899261475, + "score": -0.5520326723090689 + } + }, + "time": "0:15:49.877177", + "diversity": { + "train": { + "mean_centered_corr": -0.020127171650528908, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07589761912822723 + }, + "val": { + "mean_centered_corr": -0.02017100341618061, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07972607016563416 + }, + "test": { + "mean_centered_corr": -0.0202227383852005, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07740911841392517 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 7, + 5, + 29, + 15, + 9, + 31, + 19, + 11, + 8, + 21, + 5, + 13, + 28, + 2, + 20, + 12, + 30, + 31, + 22, + 5 + ], + "head_selection_time": "0:00:00.680873" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..4b5128852 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.001-best-head-evaluation/0/report.json @@ -0,0 +1,178 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.001-evaluation/_esam_diagnostics/esam_rho0.001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5183242601978423, + "mae": 0.41130080819129944, + "r2": 0.49644213914871216, + "score": -0.5183242601978423 + }, + "val": { + "rmse": 0.5577754983463842, + "mae": 0.4452954828739166, + "r2": 0.41332608461380005, + "score": -0.5577754983463842 + }, + "test": { + "rmse": 0.5539871683993082, + "mae": 0.44847947359085083, + "r2": 0.4281154274940491, + "score": -0.5539871683993082 + } + }, + "time": "0:11:17.118588", + "diversity": { + "train": { + "mean_centered_corr": -0.0204029381275177, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05960493162274361 + }, + "val": { + "mean_centered_corr": -0.020754853263497353, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06137089803814888 + }, + "test": { + "mean_centered_corr": -0.020418910309672356, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06040605902671814 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 10 + ], + "head_selection_time": "0:00:00.023248" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.001-evaluation/0.toml new file mode 100644 index 000000000..f15292a8e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.001-evaluation/0.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.001 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.001-evaluation/0/report.json new file mode 100644 index 000000000..bca997f7f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.001-evaluation/0/report.json @@ -0,0 +1,174 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.001-evaluation/_esam_diagnostics/esam_rho0.001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5136864373641319, + "mae": 0.4031018316745758, + "r2": 0.5054132342338562, + "score": -0.5136864373641319 + }, + "val": { + "rmse": 0.5561053998926777, + "mae": 0.44030529260635376, + "r2": 0.41683411598205566, + "score": -0.5561053998926777 + }, + "test": { + "rmse": 0.5493164333767354, + "mae": 0.44202545285224915, + "r2": 0.4377179741859436, + "score": -0.5493164333767354 + } + }, + "time": "0:11:17.118588", + "diversity": { + "train": { + "mean_centered_corr": -0.0204029381275177, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05960493162274361 + }, + "val": { + "mean_centered_corr": -0.020754853263497353, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06137089803814888 + }, + "test": { + "mean_centered_corr": -0.020418910309672356, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06040605902671814 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..92ef47e51 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,187 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.001-evaluation/_esam_diagnostics/esam_rho0.001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5139899193910727, + "mae": 0.40505242347717285, + "r2": 0.504828691482544, + "score": -0.5139899193910727 + }, + "val": { + "rmse": 0.5555279503030305, + "mae": 0.4421366751194, + "r2": 0.4180445671081543, + "score": -0.5555279503030305 + }, + "test": { + "rmse": 0.5511287648017675, + "mae": 0.4457325041294098, + "r2": 0.4340016841888428, + "score": -0.5511287648017675 + } + }, + "time": "0:11:17.118588", + "diversity": { + "train": { + "mean_centered_corr": -0.0204029381275177, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05960493162274361 + }, + "val": { + "mean_centered_corr": -0.020754853263497353, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06137089803814888 + }, + "test": { + "mean_centered_corr": -0.020418910309672356, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06040605902671814 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 10, + 21, + 16, + 0, + 15, + 22, + 6, + 11, + 19, + 7 + ], + "head_selection_time": "0:00:00.291162" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.0025-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.0025-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.0025-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.0025-best-head-evaluation/0/report.json new file mode 100644 index 000000000..1ce08308f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.0025-best-head-evaluation/0/report.json @@ -0,0 +1,178 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.0025-evaluation/_esam_diagnostics/esam_rho0.0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5103791151700686, + "mae": 0.4032345414161682, + "r2": 0.511761486530304, + "score": -0.5103791151700686 + }, + "val": { + "rmse": 0.5594097126680496, + "mae": 0.445120632648468, + "r2": 0.4098833203315735, + "score": -0.5594097126680496 + }, + "test": { + "rmse": 0.5568830263477744, + "mae": 0.44928455352783203, + "r2": 0.42212098836898804, + "score": -0.5568830263477744 + } + }, + "time": "0:12:55.387180", + "diversity": { + "train": { + "mean_centered_corr": -0.020293518900871277, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0689454972743988 + }, + "val": { + "mean_centered_corr": -0.020340915769338608, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07138139009475708 + }, + "test": { + "mean_centered_corr": -0.02027955837547779, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07020118087530136 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 5 + ], + "head_selection_time": "0:00:00.023169" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.0025-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.0025-evaluation/0.toml new file mode 100644 index 000000000..9e739cfcb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.0025-evaluation/0.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.0025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.0025-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.0025-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.0025-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.0025-evaluation/0/report.json new file mode 100644 index 000000000..ee647a6d3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.0025-evaluation/0/report.json @@ -0,0 +1,174 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.0025-evaluation/_esam_diagnostics/esam_rho0.0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5055408727023979, + "mae": 0.399361252784729, + "r2": 0.5209742784500122, + "score": -0.5055408727023979 + }, + "val": { + "rmse": 0.5559746489354217, + "mae": 0.44288384914398193, + "r2": 0.41710829734802246, + "score": -0.5559746489354217 + }, + "test": { + "rmse": 0.5535077457230605, + "mae": 0.4476969540119171, + "r2": 0.4291048049926758, + "score": -0.5535077457230605 + } + }, + "time": "0:12:55.387180", + "diversity": { + "train": { + "mean_centered_corr": -0.020293518900871277, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0689454972743988 + }, + "val": { + "mean_centered_corr": -0.020340915769338608, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07138139009475708 + }, + "test": { + "mean_centered_corr": -0.02027955837547779, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07020118087530136 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.0025-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.0025-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.0025-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.0025-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..4bf8c5e6d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.0025-greedy-heads-evaluation/0/report.json @@ -0,0 +1,194 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.0025-evaluation/_esam_diagnostics/esam_rho0.0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5057995440474627, + "mae": 0.3991788923740387, + "r2": 0.5204839706420898, + "score": -0.5057995440474627 + }, + "val": { + "rmse": 0.5555843034564808, + "mae": 0.4417176842689514, + "r2": 0.417926549911499, + "score": -0.5555843034564808 + }, + "test": { + "rmse": 0.5528859176747987, + "mae": 0.44678282737731934, + "r2": 0.4303867816925049, + "score": -0.5528859176747987 + } + }, + "time": "0:12:55.387180", + "diversity": { + "train": { + "mean_centered_corr": -0.020293518900871277, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0689454972743988 + }, + "val": { + "mean_centered_corr": -0.020340915769338608, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07138139009475708 + }, + "test": { + "mean_centered_corr": -0.02027955837547779, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07020118087530136 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 5, + 6, + 4, + 12, + 13, + 10, + 18, + 21, + 23, + 24, + 4, + 10, + 19, + 13, + 26, + 9, + 17 + ], + "head_selection_time": "0:00:00.546898" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..c0633aa71 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.005-best-head-evaluation/0/report.json @@ -0,0 +1,178 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.005-evaluation/_esam_diagnostics/esam_rho0.005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5123514530013489, + "mae": 0.4078504741191864, + "r2": 0.5079806447029114, + "score": -0.5123514530013489 + }, + "val": { + "rmse": 0.5582863288518123, + "mae": 0.4448302686214447, + "r2": 0.41225099563598633, + "score": -0.5582863288518123 + }, + "test": { + "rmse": 0.5560390501031047, + "mae": 0.4517032206058502, + "r2": 0.4238712191581726, + "score": -0.5560390501031047 + } + }, + "time": "0:12:54.891645", + "diversity": { + "train": { + "mean_centered_corr": -0.02027146704494953, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06810352951288223 + }, + "val": { + "mean_centered_corr": -0.02014581859111786, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07032498717308044 + }, + "test": { + "mean_centered_corr": -0.02043480984866619, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0694202110171318 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023468" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.005-evaluation/0.toml new file mode 100644 index 000000000..4b93a10d8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.005-evaluation/0.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.005-evaluation/0/report.json new file mode 100644 index 000000000..616e570ec --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.005-evaluation/0/report.json @@ -0,0 +1,174 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.005-evaluation/_esam_diagnostics/esam_rho0.005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.504551673775848, + "mae": 0.39544814825057983, + "r2": 0.5228471159934998, + "score": -0.504551673775848 + }, + "val": { + "rmse": 0.5556360918188752, + "mae": 0.439733624458313, + "r2": 0.41781800985336304, + "score": -0.5556360918188752 + }, + "test": { + "rmse": 0.5494649592968895, + "mae": 0.4416595995426178, + "r2": 0.43741387128829956, + "score": -0.5494649592968895 + } + }, + "time": "0:12:54.891645", + "diversity": { + "train": { + "mean_centered_corr": -0.02027146704494953, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06810352951288223 + }, + "val": { + "mean_centered_corr": -0.02014581859111786, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07032498717308044 + }, + "test": { + "mean_centered_corr": -0.02043480984866619, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0694202110171318 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..7712e89dd --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,190 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.005-evaluation/_esam_diagnostics/esam_rho0.005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5047633843312425, + "mae": 0.3963008522987366, + "r2": 0.5224465727806091, + "score": -0.5047633843312425 + }, + "val": { + "rmse": 0.5551513290724005, + "mae": 0.4403177499771118, + "r2": 0.4188333749771118, + "score": -0.5551513290724005 + }, + "test": { + "rmse": 0.5507488105906536, + "mae": 0.44414353370666504, + "r2": 0.43478184938430786, + "score": -0.5507488105906536 + } + }, + "time": "0:12:54.891645", + "diversity": { + "train": { + "mean_centered_corr": -0.02027146704494953, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06810352951288223 + }, + "val": { + "mean_centered_corr": -0.02014581859111786, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07032498717308044 + }, + "test": { + "mean_centered_corr": -0.02043480984866619, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0694202110171318 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13, + 21, + 10, + 6, + 29, + 12, + 17, + 15, + 8, + 22, + 16, + 21, + 4 + ], + "head_selection_time": "0:00:00.401490" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..4b82b622b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.01-best-head-evaluation/0/report.json @@ -0,0 +1,178 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.01-evaluation/_esam_diagnostics/esam_rho0.01-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5210981331659499, + "mae": 0.4159780442714691, + "r2": 0.49103808403015137, + "score": -0.5210981331659499 + }, + "val": { + "rmse": 0.5577866919608585, + "mae": 0.44278469681739807, + "r2": 0.41330254077911377, + "score": -0.5577866919608585 + }, + "test": { + "rmse": 0.5538906498744642, + "mae": 0.44725048542022705, + "r2": 0.4283146262168884, + "score": -0.5538906498744642 + } + }, + "time": "0:11:14.241445", + "diversity": { + "train": { + "mean_centered_corr": -0.02045314945280552, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06041118502616882 + }, + "val": { + "mean_centered_corr": -0.020492304116487503, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06114945560693741 + }, + "test": { + "mean_centered_corr": -0.020352158695459366, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06005072593688965 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 29 + ], + "head_selection_time": "0:00:00.023864" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.01-evaluation/0.toml new file mode 100644 index 000000000..8a8fdc50a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.01-evaluation/0.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.01 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.01-evaluation/0/report.json new file mode 100644 index 000000000..950992b97 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.01-evaluation/0/report.json @@ -0,0 +1,174 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.01-evaluation/_esam_diagnostics/esam_rho0.01-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5141867316497518, + "mae": 0.40462085604667664, + "r2": 0.5044494271278381, + "score": -0.5141867316497518 + }, + "val": { + "rmse": 0.555695893192201, + "mae": 0.4408281147480011, + "r2": 0.4176926612854004, + "score": -0.555695893192201 + }, + "test": { + "rmse": 0.5503082950221266, + "mae": 0.44391509890556335, + "r2": 0.4356856346130371, + "score": -0.5503082950221266 + } + }, + "time": "0:11:14.241445", + "diversity": { + "train": { + "mean_centered_corr": -0.02045314945280552, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06041118502616882 + }, + "val": { + "mean_centered_corr": -0.020492304116487503, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06114945560693741 + }, + "test": { + "mean_centered_corr": -0.020352158695459366, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06005072593688965 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..b0fcbd994 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,199 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.01-evaluation/_esam_diagnostics/esam_rho0.01-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5144364211629957, + "mae": 0.40600836277008057, + "r2": 0.5039680004119873, + "score": -0.5144364211629957 + }, + "val": { + "rmse": 0.5553495995574796, + "mae": 0.4413377344608307, + "r2": 0.41841816902160645, + "score": -0.5553495995574796 + }, + "test": { + "rmse": 0.5513616169057922, + "mae": 0.4456392228603363, + "r2": 0.4335232973098755, + "score": -0.5513616169057922 + } + }, + "time": "0:11:14.241445", + "diversity": { + "train": { + "mean_centered_corr": -0.02045314945280552, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06041118502616882 + }, + "val": { + "mean_centered_corr": -0.020492304116487503, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06114945560693741 + }, + "test": { + "mean_centered_corr": -0.020352158695459366, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06005072593688965 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 29, + 18, + 2, + 16, + 15, + 8, + 6, + 22, + 21, + 10, + 11, + 15, + 8, + 19, + 25, + 6, + 16, + 26, + 8, + 15, + 22, + 14 + ], + "head_selection_time": "0:00:00.757838" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_cf_fisd-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_cf_fisd-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_cf_fisd-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_cf_fisd-best-head-evaluation/0/report.json new file mode 100644 index 000000000..d8b7f4200 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_cf_fisd-best-head-evaluation/0/report.json @@ -0,0 +1,485 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9009943008422852, + "min_features_per_member": 188.0, + "mean_features_per_member": 198.21875, + "max_features_per_member": 206.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.831817626953125, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8196317439934785, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5194237963617938, + "mae": 0.41060853004455566, + "r2": 0.49430352449417114, + "score": -0.5194237963617938 + }, + "val": { + "rmse": 0.556619663847731, + "mae": 0.44304853677749634, + "r2": 0.415755033493042, + "score": -0.556619663847731 + }, + "test": { + "rmse": 0.55593546321811, + "mae": 0.4501691162586212, + "r2": 0.4240858554840088, + "score": -0.55593546321811 + } + }, + "time": "0:08:29.241452", + "diversity": { + "train": { + "mean_centered_corr": -0.01908317767083645, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06313066184520721 + }, + "val": { + "mean_centered_corr": -0.018972836434841156, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06387995928525925 + }, + "test": { + "mean_centered_corr": -0.019194239750504494, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06254689395427704 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 25 + ], + "head_selection_time": "0:00:00.024039" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_cf_fisd-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_cf_fisd-evaluation/0.toml new file mode 100644 index 000000000..79c621548 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_cf_fisd-evaluation/0.toml @@ -0,0 +1,57 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 81 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "delivery-eta" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_cf_fisd-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_cf_fisd-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_cf_fisd-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_cf_fisd-evaluation/0/report.json new file mode 100644 index 000000000..6010d95f4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_cf_fisd-evaluation/0/report.json @@ -0,0 +1,481 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9009943008422852, + "min_features_per_member": 188.0, + "mean_features_per_member": 198.21875, + "max_features_per_member": 206.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.831817626953125, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8196317439934785, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5154945613137585, + "mae": 0.408516526222229, + "r2": 0.5019253492355347, + "score": -0.5154945613137585 + }, + "val": { + "rmse": 0.55593889408891, + "mae": 0.4436722695827484, + "r2": 0.4171832799911499, + "score": -0.55593889408891 + }, + "test": { + "rmse": 0.5537058512136184, + "mae": 0.448915034532547, + "r2": 0.4286960959434509, + "score": -0.5537058512136184 + } + }, + "time": "0:08:29.241452", + "diversity": { + "train": { + "mean_centered_corr": -0.01908317767083645, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06313066184520721 + }, + "val": { + "mean_centered_corr": -0.018972836434841156, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06387995928525925 + }, + "test": { + "mean_centered_corr": -0.019194239750504494, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06254689395427704 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_cf_fisd-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_cf_fisd-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_cf_fisd-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_cf_fisd-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..4227cc35c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_cf_fisd-greedy-heads-evaluation/0/report.json @@ -0,0 +1,494 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9009943008422852, + "min_features_per_member": 188.0, + "mean_features_per_member": 198.21875, + "max_features_per_member": 206.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.831817626953125, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8196317439934785, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5159712120423912, + "mae": 0.4090636968612671, + "r2": 0.5010038614273071, + "score": -0.5159712120423912 + }, + "val": { + "rmse": 0.5555428639260286, + "mae": 0.4416162967681885, + "r2": 0.418013334274292, + "score": -0.5555428639260286 + }, + "test": { + "rmse": 0.5513865344186518, + "mae": 0.44544583559036255, + "r2": 0.4334721565246582, + "score": -0.5513865344186518 + } + }, + "time": "0:08:29.241452", + "diversity": { + "train": { + "mean_centered_corr": -0.01908317767083645, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06313066184520721 + }, + "val": { + "mean_centered_corr": -0.018972836434841156, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06387995928525925 + }, + "test": { + "mean_centered_corr": -0.019194239750504494, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06254689395427704 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 25, + 28, + 20, + 23, + 29, + 14, + 1, + 26, + 12, + 29 + ], + "head_selection_time": "0:00:00.295432" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.7-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.7-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.7-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.7-best-head-evaluation/0/report.json new file mode 100644 index 000000000..4d0b207d0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.7-best-head-evaluation/0/report.json @@ -0,0 +1,204 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.7046874761581421, + "min_features_per_member": 142.0, + "mean_features_per_member": 155.03125, + "max_features_per_member": 168.0, + "min_members_per_feature": 16.0, + "mean_members_per_feature": 22.549999237060547, + "max_members_per_feature": 29.0, + "mean_pairwise_jaccard": 0.5446517033602734, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2730, + "metrics": { + "train": { + "rmse": 0.5070207145078325, + "mae": 0.39629772305488586, + "r2": 0.5181657075881958, + "score": -0.5070207145078325 + }, + "val": { + "rmse": 0.5595885535124743, + "mae": 0.4446312487125397, + "r2": 0.4095059037208557, + "score": -0.5595885535124743 + }, + "test": { + "rmse": 0.5534224254724212, + "mae": 0.44561949372291565, + "r2": 0.42928075790405273, + "score": -0.5534224254724212 + } + }, + "time": "0:09:13.835446", + "diversity": { + "train": { + "mean_centered_corr": 0.022733226418495178, + "mean_pairwise_disagreement": NaN, + "member_std": 0.12898561358451843 + }, + "val": { + "mean_centered_corr": 0.03887920081615448, + "mean_pairwise_disagreement": NaN, + "member_std": 0.14495275914669037 + }, + "test": { + "mean_centered_corr": 0.04169675335288048, + "mean_pairwise_disagreement": NaN, + "member_std": 0.1456397920846939 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 8 + ], + "head_selection_time": "0:00:00.023598" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.7-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.7-evaluation/0.toml new file mode 100644 index 000000000..040e3d71f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.7-evaluation/0.toml @@ -0,0 +1,46 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.7 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.7-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.7-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.7-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.7-evaluation/0/report.json new file mode 100644 index 000000000..87e75b5e0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.7-evaluation/0/report.json @@ -0,0 +1,200 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.7046874761581421, + "min_features_per_member": 142.0, + "mean_features_per_member": 155.03125, + "max_features_per_member": 168.0, + "min_members_per_feature": 16.0, + "mean_members_per_feature": 22.549999237060547, + "max_members_per_feature": 29.0, + "mean_pairwise_jaccard": 0.5446517033602734, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2730, + "metrics": { + "train": { + "rmse": 0.5028727675143494, + "mae": 0.39573734998703003, + "r2": 0.5260173082351685, + "score": -0.5028727675143494 + }, + "val": { + "rmse": 0.5576335149669441, + "mae": 0.4440232217311859, + "r2": 0.41362476348876953, + "score": -0.5576335149669441 + }, + "test": { + "rmse": 0.5507854163911843, + "mae": 0.44484779238700867, + "r2": 0.4347066879272461, + "score": -0.5507854163911843 + } + }, + "time": "0:09:13.835446", + "diversity": { + "train": { + "mean_centered_corr": 0.022733226418495178, + "mean_pairwise_disagreement": NaN, + "member_std": 0.12898561358451843 + }, + "val": { + "mean_centered_corr": 0.03887920081615448, + "mean_pairwise_disagreement": NaN, + "member_std": 0.14495275914669037 + }, + "test": { + "mean_centered_corr": 0.04169675335288048, + "mean_pairwise_disagreement": NaN, + "member_std": 0.1456397920846939 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.7-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.7-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.7-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.7-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..22735cb5d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.7-greedy-heads-evaluation/0/report.json @@ -0,0 +1,211 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.7046874761581421, + "min_features_per_member": 142.0, + "mean_features_per_member": 155.03125, + "max_features_per_member": 168.0, + "min_members_per_feature": 16.0, + "mean_members_per_feature": 22.549999237060547, + "max_members_per_feature": 29.0, + "mean_pairwise_jaccard": 0.5446517033602734, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2730, + "metrics": { + "train": { + "rmse": 0.503753402357704, + "mae": 0.3957558870315552, + "r2": 0.5243557691574097, + "score": -0.503753402357704 + }, + "val": { + "rmse": 0.5551008108444078, + "mae": 0.43975356221199036, + "r2": 0.41893917322158813, + "score": -0.5551008108444078 + }, + "test": { + "rmse": 0.5501919833980724, + "mae": 0.44342172145843506, + "r2": 0.4359241724014282, + "score": -0.5501919833980724 + } + }, + "time": "0:09:13.835446", + "diversity": { + "train": { + "mean_centered_corr": 0.022733226418495178, + "mean_pairwise_disagreement": NaN, + "member_std": 0.12898561358451843 + }, + "val": { + "mean_centered_corr": 0.03887920081615448, + "mean_pairwise_disagreement": NaN, + "member_std": 0.14495275914669037 + }, + "test": { + "mean_centered_corr": 0.04169675335288048, + "mean_pairwise_disagreement": NaN, + "member_std": 0.1456397920846939 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 8, + 11, + 4, + 16, + 2, + 7, + 23, + 18 + ], + "head_selection_time": "0:00:00.227173" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.8-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.8-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.8-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.8-best-head-evaluation/0/report.json new file mode 100644 index 000000000..26f8a4eed --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.8-best-head-evaluation/0/report.json @@ -0,0 +1,204 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.8049715757369995, + "min_features_per_member": 164.0, + "mean_features_per_member": 177.09375, + "max_features_per_member": 189.0, + "min_members_per_feature": 20.0, + "mean_members_per_feature": 25.759090423583984, + "max_members_per_feature": 31.0, + "mean_pairwise_jaccard": 0.6745446995892237, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2730, + "metrics": { + "train": { + "rmse": 0.5119150673318991, + "mae": 0.40731897950172424, + "r2": 0.5088183879852295, + "score": -0.5119150673318991 + }, + "val": { + "rmse": 0.5594878077640214, + "mae": 0.44474470615386963, + "r2": 0.40971851348876953, + "score": -0.5594878077640214 + }, + "test": { + "rmse": 0.5558040556519808, + "mae": 0.4507056474685669, + "r2": 0.4243581295013428, + "score": -0.5558040556519808 + } + }, + "time": "0:09:15.530137", + "diversity": { + "train": { + "mean_centered_corr": 0.008296106941998005, + "mean_pairwise_disagreement": NaN, + "member_std": 0.10944075882434845 + }, + "val": { + "mean_centered_corr": 0.0186319500207901, + "mean_pairwise_disagreement": NaN, + "member_std": 0.12241803109645844 + }, + "test": { + "mean_centered_corr": 0.021373353898525238, + "mean_pairwise_disagreement": NaN, + "member_std": 0.12239326536655426 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.023292" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.8-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.8-evaluation/0.toml new file mode 100644 index 000000000..4c7765d8f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.8-evaluation/0.toml @@ -0,0 +1,46 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.8 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.8-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.8-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.8-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.8-evaluation/0/report.json new file mode 100644 index 000000000..4c0521908 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.8-evaluation/0/report.json @@ -0,0 +1,200 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.8049715757369995, + "min_features_per_member": 164.0, + "mean_features_per_member": 177.09375, + "max_features_per_member": 189.0, + "min_members_per_feature": 20.0, + "mean_members_per_feature": 25.759090423583984, + "max_members_per_feature": 31.0, + "mean_pairwise_jaccard": 0.6745446995892237, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2730, + "metrics": { + "train": { + "rmse": 0.4994902993063076, + "mae": 0.3910210430622101, + "r2": 0.5323721170425415, + "score": -0.4994902993063076 + }, + "val": { + "rmse": 0.5567305910455574, + "mae": 0.4403575658798218, + "r2": 0.41552215814590454, + "score": -0.5567305910455574 + }, + "test": { + "rmse": 0.5489749915571137, + "mae": 0.4408988654613495, + "r2": 0.43841683864593506, + "score": -0.5489749915571137 + } + }, + "time": "0:09:15.530137", + "diversity": { + "train": { + "mean_centered_corr": 0.008296106941998005, + "mean_pairwise_disagreement": NaN, + "member_std": 0.10944075882434845 + }, + "val": { + "mean_centered_corr": 0.0186319500207901, + "mean_pairwise_disagreement": NaN, + "member_std": 0.12241803109645844 + }, + "test": { + "mean_centered_corr": 0.021373353898525238, + "mean_pairwise_disagreement": NaN, + "member_std": 0.12239326536655426 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.8-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.8-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.8-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.8-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..ad4c50f24 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.8-greedy-heads-evaluation/0/report.json @@ -0,0 +1,220 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.8049715757369995, + "min_features_per_member": 164.0, + "mean_features_per_member": 177.09375, + "max_features_per_member": 189.0, + "min_members_per_feature": 20.0, + "mean_members_per_feature": 25.759090423583984, + "max_members_per_feature": 31.0, + "mean_pairwise_jaccard": 0.6745446995892237, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2730, + "metrics": { + "train": { + "rmse": 0.5003759937395629, + "mae": 0.39189252257347107, + "r2": 0.5307123064994812, + "score": -0.5003759937395629 + }, + "val": { + "rmse": 0.5554990875501031, + "mae": 0.43922579288482666, + "r2": 0.4181050658226013, + "score": -0.5554990875501031 + }, + "test": { + "rmse": 0.5493161621093208, + "mae": 0.4414233863353729, + "r2": 0.43771857023239136, + "score": -0.5493161621093208 + } + }, + "time": "0:09:15.530137", + "diversity": { + "train": { + "mean_centered_corr": 0.008296106941998005, + "mean_pairwise_disagreement": NaN, + "member_std": 0.10944075882434845 + }, + "val": { + "mean_centered_corr": 0.0186319500207901, + "mean_pairwise_disagreement": NaN, + "member_std": 0.12241803109645844 + }, + "test": { + "mean_centered_corr": 0.021373353898525238, + "mean_pairwise_disagreement": NaN, + "member_std": 0.12239326536655426 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 4, + 22, + 27, + 15, + 2, + 7, + 23, + 21, + 8, + 10, + 18, + 25, + 27, + 28, + 12, + 2, + 7 + ], + "head_selection_time": "0:00:00.551471" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.9-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.9-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.9-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.9-best-head-evaluation/0/report.json new file mode 100644 index 000000000..e71b44bed --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.9-best-head-evaluation/0/report.json @@ -0,0 +1,204 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9009943008422852, + "min_features_per_member": 188.0, + "mean_features_per_member": 198.21875, + "max_features_per_member": 206.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.831817626953125, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8196317439934785, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5157195206907822, + "mae": 0.4108550548553467, + "r2": 0.501490592956543, + "score": -0.5157195206907822 + }, + "val": { + "rmse": 0.5590040394051904, + "mae": 0.4459855556488037, + "r2": 0.41073888540267944, + "score": -0.5590040394051904 + }, + "test": { + "rmse": 0.5557457406296488, + "mae": 0.45139390230178833, + "r2": 0.42447882890701294, + "score": -0.5557457406296488 + } + }, + "time": "0:08:30.539944", + "diversity": { + "train": { + "mean_centered_corr": -0.02001963183283806, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07454843819141388 + }, + "val": { + "mean_centered_corr": -0.020045898854732513, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07650969177484512 + }, + "test": { + "mean_centered_corr": -0.02007085457444191, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0762816071510315 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023644" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.9-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.9-evaluation/0.toml new file mode 100644 index 000000000..8328925cb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.9-evaluation/0.toml @@ -0,0 +1,46 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.9-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.9-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.9-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.9-evaluation/0/report.json new file mode 100644 index 000000000..dea53c836 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.9-evaluation/0/report.json @@ -0,0 +1,200 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9009943008422852, + "min_features_per_member": 188.0, + "mean_features_per_member": 198.21875, + "max_features_per_member": 206.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.831817626953125, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8196317439934785, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.505005369677181, + "mae": 0.3960992991924286, + "r2": 0.521988570690155, + "score": -0.505005369677181 + }, + "val": { + "rmse": 0.5560026829864069, + "mae": 0.4405945837497711, + "r2": 0.4170495271682739, + "score": -0.5560026829864069 + }, + "test": { + "rmse": 0.5501201530906976, + "mae": 0.4423939883708954, + "r2": 0.43607139587402344, + "score": -0.5501201530906976 + } + }, + "time": "0:08:30.539944", + "diversity": { + "train": { + "mean_centered_corr": -0.02001963183283806, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07454843819141388 + }, + "val": { + "mean_centered_corr": -0.020045898854732513, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07650969177484512 + }, + "test": { + "mean_centered_corr": -0.02007085457444191, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0762816071510315 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.9-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.9-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.9-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.9-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..aae596674 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.9-greedy-heads-evaluation/0/report.json @@ -0,0 +1,217 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9009943008422852, + "min_features_per_member": 188.0, + "mean_features_per_member": 198.21875, + "max_features_per_member": 206.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.831817626953125, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8196317439934785, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.505321820939252, + "mae": 0.3968241810798645, + "r2": 0.5213893055915833, + "score": -0.505321820939252 + }, + "val": { + "rmse": 0.5555016895512279, + "mae": 0.4406256079673767, + "r2": 0.418099582195282, + "score": -0.5555016895512279 + }, + "test": { + "rmse": 0.5503428181880856, + "mae": 0.44352859258651733, + "r2": 0.43561482429504395, + "score": -0.5503428181880856 + } + }, + "time": "0:08:30.539944", + "diversity": { + "train": { + "mean_centered_corr": -0.02001963183283806, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07454843819141388 + }, + "val": { + "mean_centered_corr": -0.020045898854732513, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07650969177484512 + }, + "test": { + "mean_centered_corr": -0.02007085457444191, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0762816071510315 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13, + 6, + 25, + 26, + 15, + 23, + 17, + 6, + 4, + 24, + 21, + 12, + 6, + 10 + ], + "head_selection_time": "0:00:00.431663" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.95-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.95-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.95-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.95-best-head-evaluation/0/report.json new file mode 100644 index 000000000..09d10242a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.95-best-head-evaluation/0/report.json @@ -0,0 +1,204 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9481534361839294, + "min_features_per_member": 201.0, + "mean_features_per_member": 208.59375, + "max_features_per_member": 216.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.340909957885742, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9014792827014309, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.51368986033004, + "mae": 0.4090161621570587, + "r2": 0.5054066777229309, + "score": -0.51368986033004 + }, + "val": { + "rmse": 0.5584568843028204, + "mae": 0.4441465139389038, + "r2": 0.4118918776512146, + "score": -0.5584568843028204 + }, + "test": { + "rmse": 0.5549022634897627, + "mae": 0.449065625667572, + "r2": 0.4262245297431946, + "score": -0.5549022634897627 + } + }, + "time": "0:08:30.501515", + "diversity": { + "train": { + "mean_centered_corr": -0.020021812990307808, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07152318209409714 + }, + "val": { + "mean_centered_corr": -0.020280156284570694, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0745302140712738 + }, + "test": { + "mean_centered_corr": -0.019991939887404442, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07362524420022964 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023937" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.95-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.95-evaluation/0.toml new file mode 100644 index 000000000..f3a888a1c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.95-evaluation/0.toml @@ -0,0 +1,46 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.95 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.95-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.95-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.95-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.95-evaluation/0/report.json new file mode 100644 index 000000000..f258aa6ac --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.95-evaluation/0/report.json @@ -0,0 +1,200 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9481534361839294, + "min_features_per_member": 201.0, + "mean_features_per_member": 208.59375, + "max_features_per_member": 216.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.340909957885742, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9014792827014309, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5051799615450683, + "mae": 0.3969549536705017, + "r2": 0.5216580033302307, + "score": -0.5051799615450683 + }, + "val": { + "rmse": 0.5557158166151223, + "mae": 0.4399119019508362, + "r2": 0.41765087842941284, + "score": -0.5557158166151223 + }, + "test": { + "rmse": 0.5496765041498511, + "mae": 0.4418066442012787, + "r2": 0.43698060512542725, + "score": -0.5496765041498511 + } + }, + "time": "0:08:30.501515", + "diversity": { + "train": { + "mean_centered_corr": -0.020021812990307808, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07152318209409714 + }, + "val": { + "mean_centered_corr": -0.020280156284570694, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0745302140712738 + }, + "test": { + "mean_centered_corr": -0.019991939887404442, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07362524420022964 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.95-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.95-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.95-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.95-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..1ca17b257 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.95-greedy-heads-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9481534361839294, + "min_features_per_member": 201.0, + "mean_features_per_member": 208.59375, + "max_features_per_member": 216.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.340909957885742, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9014792827014309, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.506557145369867, + "mae": 0.3995445668697357, + "r2": 0.519046425819397, + "score": -0.506557145369867 + }, + "val": { + "rmse": 0.5552807442811304, + "mae": 0.4407994747161865, + "r2": 0.4185624122619629, + "score": -0.5552807442811304 + }, + "test": { + "rmse": 0.5512821544083845, + "mae": 0.4446823000907898, + "r2": 0.43368661403656006, + "score": -0.5512821544083845 + } + }, + "time": "0:08:30.501515", + "diversity": { + "train": { + "mean_centered_corr": -0.020021812990307808, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07152318209409714 + }, + "val": { + "mean_centered_corr": -0.020280156284570694, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0745302140712738 + }, + "test": { + "mean_centered_corr": -0.019991939887404442, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07362524420022964 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13, + 23, + 18, + 17, + 21, + 12, + 6, + 8, + 4, + 17 + ], + "head_selection_time": "0:00:00.291365" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_cf_fisd-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_cf_fisd-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_cf_fisd-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_cf_fisd-best-head-evaluation/0/report.json new file mode 100644 index 000000000..3314112db --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_cf_fisd-best-head-evaluation/0/report.json @@ -0,0 +1,460 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5174210795978582, + "mae": 0.40967971086502075, + "r2": 0.4981955885887146, + "score": -0.5174210795978582 + }, + "val": { + "rmse": 0.5567133805762534, + "mae": 0.4420411288738251, + "r2": 0.4155582785606384, + "score": -0.5567133805762534 + }, + "test": { + "rmse": 0.5518657747493417, + "mae": 0.44505882263183594, + "r2": 0.432486891746521, + "score": -0.5518657747493417 + } + }, + "time": "0:16:08.700759", + "diversity": { + "train": { + "mean_centered_corr": -0.019840920343995094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.054146915674209595 + }, + "val": { + "mean_centered_corr": -0.01915006712079048, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0556391216814518 + }, + "test": { + "mean_centered_corr": -0.019136715680360794, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05420564487576485 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 29 + ], + "head_selection_time": "0:00:00.023179" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_cf_fisd-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_cf_fisd-evaluation/0.toml new file mode 100644 index 000000000..28ed0aa45 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_cf_fisd-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "delivery-eta" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_cf_fisd-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_cf_fisd-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_cf_fisd-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_cf_fisd-evaluation/0/report.json new file mode 100644 index 000000000..bc6b9c016 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_cf_fisd-evaluation/0/report.json @@ -0,0 +1,456 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5117054420662647, + "mae": 0.4039783775806427, + "r2": 0.5092206001281738, + "score": -0.5117054420662647 + }, + "val": { + "rmse": 0.556067616855386, + "mae": 0.4418751299381256, + "r2": 0.41691333055496216, + "score": -0.556067616855386 + }, + "test": { + "rmse": 0.5512732344334902, + "mae": 0.44450679421424866, + "r2": 0.43370485305786133, + "score": -0.5512732344334902 + } + }, + "time": "0:16:08.700759", + "diversity": { + "train": { + "mean_centered_corr": -0.019840920343995094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.054146915674209595 + }, + "val": { + "mean_centered_corr": -0.01915006712079048, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0556391216814518 + }, + "test": { + "mean_centered_corr": -0.019136715680360794, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05420564487576485 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_cf_fisd-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_cf_fisd-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_cf_fisd-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_cf_fisd-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..7d9df2857 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_cf_fisd-greedy-heads-evaluation/0/report.json @@ -0,0 +1,468 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5125765417673701, + "mae": 0.40452396869659424, + "r2": 0.5075482130050659, + "score": -0.5125765417673701 + }, + "val": { + "rmse": 0.55586820851409, + "mae": 0.44162702560424805, + "r2": 0.4173314571380615, + "score": -0.55586820851409 + }, + "test": { + "rmse": 0.5514142072129953, + "mae": 0.4448014497756958, + "r2": 0.4334152340888977, + "score": -0.5514142072129953 + } + }, + "time": "0:16:08.700759", + "diversity": { + "train": { + "mean_centered_corr": -0.019840920343995094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.054146915674209595 + }, + "val": { + "mean_centered_corr": -0.01915006712079048, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0556391216814518 + }, + "test": { + "mean_centered_corr": -0.019136715680360794, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05420564487576485 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 29, + 31, + 15, + 1, + 27, + 28, + 18, + 8, + 25 + ], + "head_selection_time": "0:00:00.261867" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_esam-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_esam-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_esam-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_esam-best-head-evaluation/0/report.json new file mode 100644 index 000000000..97ec96d27 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_esam-best-head-evaluation/0/report.json @@ -0,0 +1,189 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_esam-evaluation/_esam_diagnostics/rla_esam-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5197141511819273, + "mae": 0.41487401723861694, + "r2": 0.49373799562454224, + "score": -0.5197141511819273 + }, + "val": { + "rmse": 0.5572907783524989, + "mae": 0.4426497519016266, + "r2": 0.41434532403945923, + "score": -0.5572907783524989 + }, + "test": { + "rmse": 0.5545303498210034, + "mae": 0.44824421405792236, + "r2": 0.42699337005615234, + "score": -0.5545303498210034 + } + }, + "time": "0:25:14.112066", + "diversity": { + "train": { + "mean_centered_corr": -0.020538095384836197, + "mean_pairwise_disagreement": NaN, + "member_std": 0.056509166955947876 + }, + "val": { + "mean_centered_corr": -0.020886661484837532, + "mean_pairwise_disagreement": NaN, + "member_std": 0.058363690972328186 + }, + "test": { + "mean_centered_corr": -0.020847400650382042, + "mean_pairwise_disagreement": NaN, + "member_std": 0.057258859276771545 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 29 + ], + "head_selection_time": "0:00:00.023366" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_esam-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_esam-evaluation/0.toml new file mode 100644 index 000000000..ce432034e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_esam-evaluation/0.toml @@ -0,0 +1,50 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_esam-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_esam-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_esam-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_esam-evaluation/0/report.json new file mode 100644 index 000000000..f51c2c83d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_esam-evaluation/0/report.json @@ -0,0 +1,185 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_esam-evaluation/_esam_diagnostics/rla_esam-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5146695445761407, + "mae": 0.40757593512535095, + "r2": 0.5035183429718018, + "score": -0.5146695445761407 + }, + "val": { + "rmse": 0.5559417888696784, + "mae": 0.44435763359069824, + "r2": 0.4171772003173828, + "score": -0.5559417888696784 + }, + "test": { + "rmse": 0.5541331785404355, + "mae": 0.44994547963142395, + "r2": 0.42781394720077515, + "score": -0.5541331785404355 + } + }, + "time": "0:25:14.112066", + "diversity": { + "train": { + "mean_centered_corr": -0.020538095384836197, + "mean_pairwise_disagreement": NaN, + "member_std": 0.056509166955947876 + }, + "val": { + "mean_centered_corr": -0.020886661484837532, + "mean_pairwise_disagreement": NaN, + "member_std": 0.058363690972328186 + }, + "test": { + "mean_centered_corr": -0.020847400650382042, + "mean_pairwise_disagreement": NaN, + "member_std": 0.057258859276771545 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_esam-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_esam-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_esam-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_esam-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..5449bd7de --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_esam-greedy-heads-evaluation/0/report.json @@ -0,0 +1,196 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_esam-evaluation/_esam_diagnostics/rla_esam-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5146023695303711, + "mae": 0.4073488712310791, + "r2": 0.5036479234695435, + "score": -0.5146023695303711 + }, + "val": { + "rmse": 0.5552988041877751, + "mae": 0.44173434376716614, + "r2": 0.4185245633125305, + "score": -0.5552988041877751 + }, + "test": { + "rmse": 0.5525187688548042, + "mae": 0.44692304730415344, + "r2": 0.43114304542541504, + "score": -0.5525187688548042 + } + }, + "time": "0:25:14.112066", + "diversity": { + "train": { + "mean_centered_corr": -0.020538095384836197, + "mean_pairwise_disagreement": NaN, + "member_std": 0.056509166955947876 + }, + "val": { + "mean_centered_corr": -0.020886661484837532, + "mean_pairwise_disagreement": NaN, + "member_std": 0.058363690972328186 + }, + "test": { + "mean_centered_corr": -0.020847400650382042, + "mean_pairwise_disagreement": NaN, + "member_std": 0.057258859276771545 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 29, + 16, + 11, + 10, + 26, + 8, + 15, + 22 + ], + "head_selection_time": "0:00:00.230227" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_mfb-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_mfb-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_mfb-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_mfb-best-head-evaluation/0/report.json new file mode 100644 index 000000000..128269a30 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_mfb-best-head-evaluation/0/report.json @@ -0,0 +1,215 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9009943008422852, + "min_features_per_member": 188.0, + "mean_features_per_member": 198.21875, + "max_features_per_member": 206.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.831817626953125, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8196317439934785, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5163692824712737, + "mae": 0.40596020221710205, + "r2": 0.5002335906028748, + "score": -0.5163692824712737 + }, + "val": { + "rmse": 0.5579516581852805, + "mae": 0.44197753071784973, + "r2": 0.41295546293258667, + "score": -0.5579516581852805 + }, + "test": { + "rmse": 0.5533480523154869, + "mae": 0.44432690739631653, + "r2": 0.4294341802597046, + "score": -0.5533480523154869 + } + }, + "time": "0:16:07.995080", + "diversity": { + "train": { + "mean_centered_corr": -0.02017831802368164, + "mean_pairwise_disagreement": NaN, + "member_std": 0.062000252306461334 + }, + "val": { + "mean_centered_corr": -0.02033226191997528, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06415332108736038 + }, + "test": { + "mean_centered_corr": -0.02038513869047165, + "mean_pairwise_disagreement": NaN, + "member_std": 0.063583143055439 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 14 + ], + "head_selection_time": "0:00:00.023639" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_mfb-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_mfb-evaluation/0.toml new file mode 100644 index 000000000..cfe3ed79e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_mfb-evaluation/0.toml @@ -0,0 +1,53 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_mfb-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_mfb-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_mfb-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_mfb-evaluation/0/report.json new file mode 100644 index 000000000..b61d36bce --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_mfb-evaluation/0/report.json @@ -0,0 +1,211 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9009943008422852, + "min_features_per_member": 188.0, + "mean_features_per_member": 198.21875, + "max_features_per_member": 206.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.831817626953125, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8196317439934785, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.513211874722488, + "mae": 0.4028632938861847, + "r2": 0.5063266754150391, + "score": -0.513211874722488 + }, + "val": { + "rmse": 0.5559017430643394, + "mae": 0.4410461187362671, + "r2": 0.41726118326187134, + "score": -0.5559017430643394 + }, + "test": { + "rmse": 0.5492186599232151, + "mae": 0.44233980774879456, + "r2": 0.43791818618774414, + "score": -0.5492186599232151 + } + }, + "time": "0:16:07.995080", + "diversity": { + "train": { + "mean_centered_corr": -0.02017831802368164, + "mean_pairwise_disagreement": NaN, + "member_std": 0.062000252306461334 + }, + "val": { + "mean_centered_corr": -0.02033226191997528, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06415332108736038 + }, + "test": { + "mean_centered_corr": -0.02038513869047165, + "mean_pairwise_disagreement": NaN, + "member_std": 0.063583143055439 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_mfb-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_mfb-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_mfb-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_mfb-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..355bf260f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_mfb-greedy-heads-evaluation/0/report.json @@ -0,0 +1,221 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9009943008422852, + "min_features_per_member": 188.0, + "mean_features_per_member": 198.21875, + "max_features_per_member": 206.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.831817626953125, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8196317439934785, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5135637466863064, + "mae": 0.40484777092933655, + "r2": 0.5056494474411011, + "score": -0.5135637466863064 + }, + "val": { + "rmse": 0.5554524372350571, + "mae": 0.4416925311088562, + "r2": 0.4182027578353882, + "score": -0.5554524372350571 + }, + "test": { + "rmse": 0.5505269863779684, + "mae": 0.4447595179080963, + "r2": 0.4352370500564575, + "score": -0.5505269863779684 + } + }, + "time": "0:16:07.995080", + "diversity": { + "train": { + "mean_centered_corr": -0.02017831802368164, + "mean_pairwise_disagreement": NaN, + "member_std": 0.062000252306461334 + }, + "val": { + "mean_centered_corr": -0.02033226191997528, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06415332108736038 + }, + "test": { + "mean_centered_corr": -0.02038513869047165, + "mean_pairwise_disagreement": NaN, + "member_std": 0.063583143055439 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 14, + 6, + 11, + 8, + 16, + 15, + 22 + ], + "head_selection_time": "0:00:00.201384" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.0001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.0001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.0001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.0001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..472f66269 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.0001-best-head-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5172845259639738, + "mae": 0.4081471264362335, + "r2": 0.49846041202545166, + "score": -0.5172845259639738 + }, + "val": { + "rmse": 0.5574308976082757, + "mae": 0.44358816742897034, + "r2": 0.41405075788497925, + "score": -0.5574308976082757 + }, + "test": { + "rmse": 0.5550464489547554, + "mae": 0.449802041053772, + "r2": 0.4259263277053833, + "score": -0.5550464489547554 + } + }, + "time": "0:09:32.457201", + "diversity": { + "train": { + "mean_centered_corr": -0.020586222410202026, + "mean_pairwise_disagreement": NaN, + "member_std": 0.057862311601638794 + }, + "val": { + "mean_centered_corr": -0.020692767575383186, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05938911437988281 + }, + "test": { + "mean_centered_corr": -0.02092323824763298, + "mean_pairwise_disagreement": NaN, + "member_std": 0.058085475116968155 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 21 + ], + "head_selection_time": "0:00:00.023792" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.0001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.0001-evaluation/0.toml new file mode 100644 index 000000000..50e88d179 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.0001-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.0001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.0001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.0001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.0001-evaluation/0/report.json new file mode 100644 index 000000000..ee601d869 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.0001-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5137669870487251, + "mae": 0.4040716290473938, + "r2": 0.5052582025527954, + "score": -0.5137669870487251 + }, + "val": { + "rmse": 0.5558827645265865, + "mae": 0.4413273334503174, + "r2": 0.4173009395599365, + "score": -0.5558827645265865 + }, + "test": { + "rmse": 0.5504866820118107, + "mae": 0.44446638226509094, + "r2": 0.435319721698761, + "score": -0.5504866820118107 + } + }, + "time": "0:09:32.457201", + "diversity": { + "train": { + "mean_centered_corr": -0.020586222410202026, + "mean_pairwise_disagreement": NaN, + "member_std": 0.057862311601638794 + }, + "val": { + "mean_centered_corr": -0.020692767575383186, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05938911437988281 + }, + "test": { + "mean_centered_corr": -0.02092323824763298, + "mean_pairwise_disagreement": NaN, + "member_std": 0.058085475116968155 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.0001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.0001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.0001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.0001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..c0804ba27 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.0001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,188 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5145618575810417, + "mae": 0.40636324882507324, + "r2": 0.503726065158844, + "score": -0.5145618575810417 + }, + "val": { + "rmse": 0.5553755991930566, + "mae": 0.44143426418304443, + "r2": 0.41836369037628174, + "score": -0.5553755991930566 + }, + "test": { + "rmse": 0.5516112816611277, + "mae": 0.4461708664894104, + "r2": 0.43301016092300415, + "score": -0.5516112816611277 + } + }, + "time": "0:09:32.457201", + "diversity": { + "train": { + "mean_centered_corr": -0.020586222410202026, + "mean_pairwise_disagreement": NaN, + "member_std": 0.057862311601638794 + }, + "val": { + "mean_centered_corr": -0.020692767575383186, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05938911437988281 + }, + "test": { + "mean_centered_corr": -0.02092323824763298, + "mean_pairwise_disagreement": NaN, + "member_std": 0.058085475116968155 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 21, + 15, + 31, + 16, + 29, + 10, + 13, + 11, + 26, + 14 + ], + "head_selection_time": "0:00:00.289768" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..c496cfa72 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.001-best-head-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5169435461643365, + "mae": 0.4078221619129181, + "r2": 0.49912136793136597, + "score": -0.5169435461643365 + }, + "val": { + "rmse": 0.5574634025946379, + "mae": 0.44402608275413513, + "r2": 0.41398245096206665, + "score": -0.5574634025946379 + }, + "test": { + "rmse": 0.5547066537478277, + "mae": 0.450179785490036, + "r2": 0.4266290068626404, + "score": -0.5547066537478277 + } + }, + "time": "0:09:33.111358", + "diversity": { + "train": { + "mean_centered_corr": -0.020444847643375397, + "mean_pairwise_disagreement": NaN, + "member_std": 0.059093035757541656 + }, + "val": { + "mean_centered_corr": -0.020575184375047684, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06087604537606239 + }, + "test": { + "mean_centered_corr": -0.020676201209425926, + "mean_pairwise_disagreement": NaN, + "member_std": 0.060276810079813004 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 18 + ], + "head_selection_time": "0:00:00.023373" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.001-evaluation/0.toml new file mode 100644 index 000000000..6d6f2bc2b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.001-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.001-evaluation/0/report.json new file mode 100644 index 000000000..03a93045c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.001-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5137012893899691, + "mae": 0.40439915657043457, + "r2": 0.5053846836090088, + "score": -0.5137012893899691 + }, + "val": { + "rmse": 0.5556336245387727, + "mae": 0.4420306384563446, + "r2": 0.4178231954574585, + "score": -0.5556336245387727 + }, + "test": { + "rmse": 0.5518591323593222, + "mae": 0.4463585913181305, + "r2": 0.43250054121017456, + "score": -0.5518591323593222 + } + }, + "time": "0:09:33.111358", + "diversity": { + "train": { + "mean_centered_corr": -0.020444847643375397, + "mean_pairwise_disagreement": NaN, + "member_std": 0.059093035757541656 + }, + "val": { + "mean_centered_corr": -0.020575184375047684, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06087604537606239 + }, + "test": { + "mean_centered_corr": -0.020676201209425926, + "mean_pairwise_disagreement": NaN, + "member_std": 0.060276810079813004 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..882c6c418 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,194 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.514097726188904, + "mae": 0.4052426517009735, + "r2": 0.5046209692955017, + "score": -0.514097726188904 + }, + "val": { + "rmse": 0.5552720495549479, + "mae": 0.441690057516098, + "r2": 0.4185805916786194, + "score": -0.5552720495549479 + }, + "test": { + "rmse": 0.5524877260752137, + "mae": 0.447174072265625, + "r2": 0.43120694160461426, + "score": -0.5524877260752137 + } + }, + "time": "0:09:33.111358", + "diversity": { + "train": { + "mean_centered_corr": -0.020444847643375397, + "mean_pairwise_disagreement": NaN, + "member_std": 0.059093035757541656 + }, + "val": { + "mean_centered_corr": -0.020575184375047684, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06087604537606239 + }, + "test": { + "mean_centered_corr": -0.020676201209425926, + "mean_pairwise_disagreement": NaN, + "member_std": 0.060276810079813004 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 18, + 10, + 11, + 26, + 21, + 14, + 31, + 19, + 11, + 16, + 26, + 10, + 11, + 15, + 21, + 29 + ], + "head_selection_time": "0:00:00.514504" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise1e-05-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise1e-05-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise1e-05-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise1e-05-best-head-evaluation/0/report.json new file mode 100644 index 000000000..afb9cebaa --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise1e-05-best-head-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5198625067814638, + "mae": 0.4135857820510864, + "r2": 0.4934489130973816, + "score": -0.5198625067814638 + }, + "val": { + "rmse": 0.5574473374558679, + "mae": 0.44392451643943787, + "r2": 0.4140162467956543, + "score": -0.5574473374558679 + }, + "test": { + "rmse": 0.5570882235759158, + "mae": 0.45243602991104126, + "r2": 0.4216949939727783, + "score": -0.5570882235759158 + } + }, + "time": "0:09:32.630353", + "diversity": { + "train": { + "mean_centered_corr": -0.02049259841442108, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05744750052690506 + }, + "val": { + "mean_centered_corr": -0.020656166598200798, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05917323753237724 + }, + "test": { + "mean_centered_corr": -0.020809270441532135, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05833838880062103 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 14 + ], + "head_selection_time": "0:00:00.023420" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise1e-05-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise1e-05-evaluation/0.toml new file mode 100644 index 000000000..0d48e7096 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise1e-05-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise1e-05-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise1e-05-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise1e-05-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise1e-05-evaluation/0/report.json new file mode 100644 index 000000000..bc695d15f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise1e-05-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5135076862361688, + "mae": 0.40329840779304504, + "r2": 0.5057573914527893, + "score": -0.5135076862361688 + }, + "val": { + "rmse": 0.5559787764083384, + "mae": 0.4406217336654663, + "r2": 0.41709965467453003, + "score": -0.5559787764083384 + }, + "test": { + "rmse": 0.5500968306007196, + "mae": 0.4433589279651642, + "r2": 0.4361191987991333, + "score": -0.5500968306007196 + } + }, + "time": "0:09:32.630353", + "diversity": { + "train": { + "mean_centered_corr": -0.02049259841442108, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05744750052690506 + }, + "val": { + "mean_centered_corr": -0.020656166598200798, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05917323753237724 + }, + "test": { + "mean_centered_corr": -0.020809270441532135, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05833838880062103 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise1e-05-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise1e-05-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise1e-05-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise1e-05-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..81009d573 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise1e-05-greedy-heads-evaluation/0/report.json @@ -0,0 +1,194 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5135710584677824, + "mae": 0.40422096848487854, + "r2": 0.5056354403495789, + "score": -0.5135710584677824 + }, + "val": { + "rmse": 0.5553479091365543, + "mae": 0.44104957580566406, + "r2": 0.41842174530029297, + "score": -0.5553479091365543 + }, + "test": { + "rmse": 0.5511482044438852, + "mae": 0.4451128840446472, + "r2": 0.43396174907684326, + "score": -0.5511482044438852 + } + }, + "time": "0:09:32.630353", + "diversity": { + "train": { + "mean_centered_corr": -0.02049259841442108, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05744750052690506 + }, + "val": { + "mean_centered_corr": -0.020656166598200798, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05917323753237724 + }, + "test": { + "mean_centered_corr": -0.020809270441532135, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05833838880062103 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 14, + 29, + 15, + 11, + 19, + 31, + 21, + 2, + 15, + 19, + 22, + 29, + 15, + 26, + 13, + 8 + ], + "head_selection_time": "0:00:00.512283" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.0001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.0001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.0001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.0001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..19557f15f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.0001-best-head-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5153977731507096, + "mae": 0.4045717418193817, + "r2": 0.5021123886108398, + "score": -0.5153977731507096 + }, + "val": { + "rmse": 0.5573103773832353, + "mae": 0.44244086742401123, + "r2": 0.41430413722991943, + "score": -0.5573103773832353 + }, + "test": { + "rmse": 0.5532468973961396, + "mae": 0.4463353753089905, + "r2": 0.4296427369117737, + "score": -0.5532468973961396 + } + }, + "time": "0:14:41.500927", + "diversity": { + "train": { + "mean_centered_corr": -0.02064315602183342, + "mean_pairwise_disagreement": NaN, + "member_std": 0.056059885770082474 + }, + "val": { + "mean_centered_corr": -0.020947575569152832, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05900698900222778 + }, + "test": { + "mean_centered_corr": -0.020849451422691345, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05796251446008682 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 31 + ], + "head_selection_time": "0:00:00.023481" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.0001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.0001-evaluation/0.toml new file mode 100644 index 000000000..9dd7c431e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.0001-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.0001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.0001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.0001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.0001-evaluation/0/report.json new file mode 100644 index 000000000..80b660117 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.0001-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5133709914722272, + "mae": 0.4036697745323181, + "r2": 0.5060205459594727, + "score": -0.5133709914722272 + }, + "val": { + "rmse": 0.5556926485294449, + "mae": 0.4411713182926178, + "r2": 0.4176994562149048, + "score": -0.5556926485294449 + }, + "test": { + "rmse": 0.5502675684403805, + "mae": 0.4438808858394623, + "r2": 0.43576914072036743, + "score": -0.5502675684403805 + } + }, + "time": "0:14:41.500927", + "diversity": { + "train": { + "mean_centered_corr": -0.02064315602183342, + "mean_pairwise_disagreement": NaN, + "member_std": 0.056059885770082474 + }, + "val": { + "mean_centered_corr": -0.020947575569152832, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05900698900222778 + }, + "test": { + "mean_centered_corr": -0.020849451422691345, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05796251446008682 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.0001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.0001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.0001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.0001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..855434aa0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.0001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,184 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5137274534378229, + "mae": 0.40429580211639404, + "r2": 0.5053342580795288, + "score": -0.5137274534378229 + }, + "val": { + "rmse": 0.5553051102575359, + "mae": 0.4410802721977234, + "r2": 0.4185113310813904, + "score": -0.5553051102575359 + }, + "test": { + "rmse": 0.5508335169889497, + "mae": 0.4448656141757965, + "r2": 0.43460792303085327, + "score": -0.5508335169889497 + } + }, + "time": "0:14:41.500927", + "diversity": { + "train": { + "mean_centered_corr": -0.02064315602183342, + "mean_pairwise_disagreement": NaN, + "member_std": 0.056059885770082474 + }, + "val": { + "mean_centered_corr": -0.020947575569152832, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05900698900222778 + }, + "test": { + "mean_centered_corr": -0.020849451422691345, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05796251446008682 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 31, + 16, + 19, + 15, + 11, + 21 + ], + "head_selection_time": "0:00:00.177208" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..13d5060bf --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.001-best-head-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5156084982803489, + "mae": 0.40666115283966064, + "r2": 0.5017051696777344, + "score": -0.5156084982803489 + }, + "val": { + "rmse": 0.5571872637155045, + "mae": 0.44240155816078186, + "r2": 0.4145628809928894, + "score": -0.5571872637155045 + }, + "test": { + "rmse": 0.5515561165588831, + "mae": 0.444357693195343, + "r2": 0.4331235885620117, + "score": -0.5515561165588831 + } + }, + "time": "0:14:43.936882", + "diversity": { + "train": { + "mean_centered_corr": -0.020564058795571327, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05423367768526077 + }, + "val": { + "mean_centered_corr": -0.020864037796854973, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05605003982782364 + }, + "test": { + "mean_centered_corr": -0.02069844678044319, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05536308139562607 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 10 + ], + "head_selection_time": "0:00:00.023846" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.001-evaluation/0.toml new file mode 100644 index 000000000..7539c979e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.001-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.001-evaluation/0/report.json new file mode 100644 index 000000000..00fcfd604 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.001-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5131794414842274, + "mae": 0.40165284276008606, + "r2": 0.5063890218734741, + "score": -0.5131794414842274 + }, + "val": { + "rmse": 0.5564461619356259, + "mae": 0.4396740794181824, + "r2": 0.41611921787261963, + "score": -0.5564461619356259 + }, + "test": { + "rmse": 0.5491469465177617, + "mae": 0.44128578901290894, + "r2": 0.43806493282318115, + "score": -0.5491469465177617 + } + }, + "time": "0:14:43.936882", + "diversity": { + "train": { + "mean_centered_corr": -0.020564058795571327, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05423367768526077 + }, + "val": { + "mean_centered_corr": -0.020864037796854973, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05605003982782364 + }, + "test": { + "mean_centered_corr": -0.02069844678044319, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05536308139562607 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..9c5138cde --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,190 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5133235315939391, + "mae": 0.4035474956035614, + "r2": 0.5061118602752686, + "score": -0.5133235315939391 + }, + "val": { + "rmse": 0.5552950473462183, + "mae": 0.44067850708961487, + "r2": 0.41853243112564087, + "score": -0.5552950473462183 + }, + "test": { + "rmse": 0.5512393912767233, + "mae": 0.44479161500930786, + "r2": 0.433774471282959, + "score": -0.5512393912767233 + } + }, + "time": "0:14:43.936882", + "diversity": { + "train": { + "mean_centered_corr": -0.020564058795571327, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05423367768526077 + }, + "val": { + "mean_centered_corr": -0.020864037796854973, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05605003982782364 + }, + "test": { + "mean_centered_corr": -0.02069844678044319, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05536308139562607 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 10, + 13, + 15, + 31, + 19, + 21, + 14, + 15, + 6, + 29, + 31, + 13 + ], + "head_selection_time": "0:00:00.354949" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise1e-05-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise1e-05-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise1e-05-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise1e-05-best-head-evaluation/0/report.json new file mode 100644 index 000000000..d3e620806 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise1e-05-best-head-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.517674217660727, + "mae": 0.41062942147254944, + "r2": 0.4977044463157654, + "score": -0.517674217660727 + }, + "val": { + "rmse": 0.5569447272303437, + "mae": 0.4422590136528015, + "r2": 0.4150724411010742, + "score": -0.5569447272303437 + }, + "test": { + "rmse": 0.5529624818746817, + "mae": 0.446209579706192, + "r2": 0.4302290081977844, + "score": -0.5529624818746817 + } + }, + "time": "0:14:43.584777", + "diversity": { + "train": { + "mean_centered_corr": -0.020540937781333923, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05564684420824051 + }, + "val": { + "mean_centered_corr": -0.02083873376250267, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05753690004348755 + }, + "test": { + "mean_centered_corr": -0.020769651979207993, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05645184963941574 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 29 + ], + "head_selection_time": "0:00:00.023450" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise1e-05-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise1e-05-evaluation/0.toml new file mode 100644 index 000000000..ae93dce05 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise1e-05-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise1e-05-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise1e-05-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise1e-05-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise1e-05-evaluation/0/report.json new file mode 100644 index 000000000..f46125d6f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise1e-05-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5139217276720119, + "mae": 0.4053196907043457, + "r2": 0.5049600601196289, + "score": -0.5139217276720119 + }, + "val": { + "rmse": 0.5560204514474498, + "mae": 0.4419814646244049, + "r2": 0.4170122742652893, + "score": -0.5560204514474498 + }, + "test": { + "rmse": 0.5515690033180755, + "mae": 0.4455237090587616, + "r2": 0.43309706449508667, + "score": -0.5515690033180755 + } + }, + "time": "0:14:43.584777", + "diversity": { + "train": { + "mean_centered_corr": -0.020540937781333923, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05564684420824051 + }, + "val": { + "mean_centered_corr": -0.02083873376250267, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05753690004348755 + }, + "test": { + "mean_centered_corr": -0.020769651979207993, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05645184963941574 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise1e-05-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise1e-05-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise1e-05-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise1e-05-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..b3e418d18 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise1e-05-greedy-heads-evaluation/0/report.json @@ -0,0 +1,187 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5139979208874492, + "mae": 0.4061751067638397, + "r2": 0.5048133134841919, + "score": -0.5139979208874492 + }, + "val": { + "rmse": 0.5553112552523684, + "mae": 0.4416539669036865, + "r2": 0.4184984564781189, + "score": -0.5553112552523684 + }, + "test": { + "rmse": 0.552273210826067, + "mae": 0.4467461109161377, + "r2": 0.4316486120223999, + "score": -0.552273210826067 + } + }, + "time": "0:14:43.584777", + "diversity": { + "train": { + "mean_centered_corr": -0.020540937781333923, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05564684420824051 + }, + "val": { + "mean_centered_corr": -0.02083873376250267, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05753690004348755 + }, + "test": { + "mean_centered_corr": -0.020769651979207993, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05645184963941574 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 29, + 10, + 26, + 19, + 11, + 15, + 21, + 13, + 31 + ], + "head_selection_time": "0:00:00.259576" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.0001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.0001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.0001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.0001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..a2fb24fa6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.0001-best-head-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 8830112, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5142542507513354, + "mae": 0.4036604166030884, + "r2": 0.5043192505836487, + "score": -0.5142542507513354 + }, + "val": { + "rmse": 0.5573390661266857, + "mae": 0.4425854980945587, + "r2": 0.41424381732940674, + "score": -0.5573390661266857 + }, + "test": { + "rmse": 0.5535791095517189, + "mae": 0.44649070501327515, + "r2": 0.42895758152008057, + "score": -0.5535791095517189 + } + }, + "time": "0:25:16.790683", + "diversity": { + "train": { + "mean_centered_corr": -0.02050994336605072, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0549476183950901 + }, + "val": { + "mean_centered_corr": -0.020625509321689606, + "mean_pairwise_disagreement": NaN, + "member_std": 0.056770239025354385 + }, + "test": { + "mean_centered_corr": -0.020527120679616928, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05573471263051033 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023311" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.0001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.0001-evaluation/0.toml new file mode 100644 index 000000000..a7ff216da --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.0001-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 8 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.0001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.0001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.0001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.0001-evaluation/0/report.json new file mode 100644 index 000000000..1078aa1c1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.0001-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 8830112, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5118505584787533, + "mae": 0.39745861291885376, + "r2": 0.5089422464370728, + "score": -0.5118505584787533 + }, + "val": { + "rmse": 0.5572424061712593, + "mae": 0.43794944882392883, + "r2": 0.41444700956344604, + "score": -0.5572424061712593 + }, + "test": { + "rmse": 0.546917368746242, + "mae": 0.43608883023262024, + "r2": 0.4426186680793762, + "score": -0.546917368746242 + } + }, + "time": "0:25:16.790683", + "diversity": { + "train": { + "mean_centered_corr": -0.02050994336605072, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0549476183950901 + }, + "val": { + "mean_centered_corr": -0.020625509321689606, + "mean_pairwise_disagreement": NaN, + "member_std": 0.056770239025354385 + }, + "test": { + "mean_centered_corr": -0.020527120679616928, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05573471263051033 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.0001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.0001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.0001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.0001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..8bc286149 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.0001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,185 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 8830112, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5124635881189222, + "mae": 0.4014242887496948, + "r2": 0.5077651739120483, + "score": -0.5124635881189222 + }, + "val": { + "rmse": 0.5558846409645507, + "mae": 0.440949946641922, + "r2": 0.41729700565338135, + "score": -0.5558846409645507 + }, + "test": { + "rmse": 0.5506559188769905, + "mae": 0.4435808062553406, + "r2": 0.43497246503829956, + "score": -0.5506559188769905 + } + }, + "time": "0:25:16.790683", + "diversity": { + "train": { + "mean_centered_corr": -0.02050994336605072, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0549476183950901 + }, + "val": { + "mean_centered_corr": -0.020625509321689606, + "mean_pairwise_disagreement": NaN, + "member_std": 0.056770239025354385 + }, + "test": { + "mean_centered_corr": -0.020527120679616928, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05573471263051033 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 13, + 21, + 15, + 14, + 6, + 21, + 7 + ], + "head_selection_time": "0:00:00.201063" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..56c3d50b2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.001-best-head-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 8830112, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5108046192647389, + "mae": 0.3996152877807617, + "r2": 0.5109469890594482, + "score": -0.5108046192647389 + }, + "val": { + "rmse": 0.5576981787979653, + "mae": 0.4411272704601288, + "r2": 0.4134887456893921, + "score": -0.5576981787979653 + }, + "test": { + "rmse": 0.5514806270620619, + "mae": 0.4422125220298767, + "r2": 0.43327873945236206, + "score": -0.5514806270620619 + } + }, + "time": "0:26:26.010094", + "diversity": { + "train": { + "mean_centered_corr": -0.020078139379620552, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05760448798537254 + }, + "val": { + "mean_centered_corr": -0.0201308261603117, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05999069660902023 + }, + "test": { + "mean_centered_corr": -0.02021006867289543, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05763765797019005 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023898" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.001-evaluation/0.toml new file mode 100644 index 000000000..cacd21376 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.001-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 8 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.001-evaluation/0/report.json new file mode 100644 index 000000000..13fcff92d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.001-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 8830112, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5079285415698682, + "mae": 0.3954585790634155, + "r2": 0.5164387822151184, + "score": -0.5079285415698682 + }, + "val": { + "rmse": 0.5561609443354546, + "mae": 0.43867626786231995, + "r2": 0.41671764850616455, + "score": -0.5561609443354546 + }, + "test": { + "rmse": 0.5482846212731874, + "mae": 0.4379252791404724, + "r2": 0.43982839584350586, + "score": -0.5482846212731874 + } + }, + "time": "0:26:26.010094", + "diversity": { + "train": { + "mean_centered_corr": -0.020078139379620552, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05760448798537254 + }, + "val": { + "mean_centered_corr": -0.0201308261603117, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05999069660902023 + }, + "test": { + "mean_centered_corr": -0.02021006867289543, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05763765797019005 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..398031dcf --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,186 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 8830112, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.508383180826817, + "mae": 0.39700230956077576, + "r2": 0.5155726671218872, + "score": -0.508383180826817 + }, + "val": { + "rmse": 0.5556463362777673, + "mae": 0.43915992975234985, + "r2": 0.4177964925765991, + "score": -0.5556463362777673 + }, + "test": { + "rmse": 0.5491140035096563, + "mae": 0.4403056800365448, + "r2": 0.4381323456764221, + "score": -0.5491140035096563 + } + }, + "time": "0:26:26.010094", + "diversity": { + "train": { + "mean_centered_corr": -0.020078139379620552, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05760448798537254 + }, + "val": { + "mean_centered_corr": -0.0201308261603117, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05999069660902023 + }, + "test": { + "mean_centered_corr": -0.02021006867289543, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05763765797019005 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 13, + 4, + 21, + 25, + 15, + 0, + 1, + 17 + ], + "head_selection_time": "0:00:00.231882" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise1e-05-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise1e-05-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise1e-05-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise1e-05-best-head-evaluation/0/report.json new file mode 100644 index 000000000..6ffea1cff --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise1e-05-best-head-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 8830112, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5162090981285068, + "mae": 0.4077680706977844, + "r2": 0.5005435943603516, + "score": -0.5162090981285068 + }, + "val": { + "rmse": 0.5574517747950514, + "mae": 0.44307076930999756, + "r2": 0.41400688886642456, + "score": -0.5574517747950514 + }, + "test": { + "rmse": 0.5534907850292722, + "mae": 0.4466339945793152, + "r2": 0.42913979291915894, + "score": -0.5534907850292722 + } + }, + "time": "0:25:18.954003", + "diversity": { + "train": { + "mean_centered_corr": -0.020567411556839943, + "mean_pairwise_disagreement": NaN, + "member_std": 0.056090690195560455 + }, + "val": { + "mean_centered_corr": -0.020773332566022873, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0600193589925766 + }, + "test": { + "mean_centered_corr": -0.020730366930365562, + "mean_pairwise_disagreement": NaN, + "member_std": 0.058352041989564896 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 10 + ], + "head_selection_time": "0:00:00.023195" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise1e-05-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise1e-05-evaluation/0.toml new file mode 100644 index 000000000..7baa58a72 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise1e-05-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 8 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise1e-05-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise1e-05-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise1e-05-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise1e-05-evaluation/0/report.json new file mode 100644 index 000000000..6f7bf2e35 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise1e-05-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 8830112, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5125091214647594, + "mae": 0.4022916555404663, + "r2": 0.5076777338981628, + "score": -0.5125091214647594 + }, + "val": { + "rmse": 0.5558005167077797, + "mae": 0.44092121720314026, + "r2": 0.41747337579727173, + "score": -0.5558005167077797 + }, + "test": { + "rmse": 0.5500678724978622, + "mae": 0.4430854618549347, + "r2": 0.43617862462997437, + "score": -0.5500678724978622 + } + }, + "time": "0:25:18.954003", + "diversity": { + "train": { + "mean_centered_corr": -0.020567411556839943, + "mean_pairwise_disagreement": NaN, + "member_std": 0.056090690195560455 + }, + "val": { + "mean_centered_corr": -0.020773332566022873, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0600193589925766 + }, + "test": { + "mean_centered_corr": -0.020730366930365562, + "mean_pairwise_disagreement": NaN, + "member_std": 0.058352041989564896 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise1e-05-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise1e-05-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise1e-05-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise1e-05-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..c28c6f984 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise1e-05-greedy-heads-evaluation/0/report.json @@ -0,0 +1,187 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 8830112, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5134856027741057, + "mae": 0.40518614649772644, + "r2": 0.5057998895645142, + "score": -0.5134856027741057 + }, + "val": { + "rmse": 0.5553793554897237, + "mae": 0.44185882806777954, + "r2": 0.4183558225631714, + "score": -0.5553793554897237 + }, + "test": { + "rmse": 0.5521969288494308, + "mae": 0.4467563331127167, + "r2": 0.4318056106567383, + "score": -0.5521969288494308 + } + }, + "time": "0:25:18.954003", + "diversity": { + "train": { + "mean_centered_corr": -0.020567411556839943, + "mean_pairwise_disagreement": NaN, + "member_std": 0.056090690195560455 + }, + "val": { + "mean_centered_corr": -0.020773332566022873, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0600193589925766 + }, + "test": { + "mean_centered_corr": -0.020730366930365562, + "mean_pairwise_disagreement": NaN, + "member_std": 0.058352041989564896 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 10, + 22, + 31, + 21, + 29, + 7, + 15, + 16, + 13 + ], + "head_selection_time": "0:00:00.257032" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_four_combined-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_four_combined-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_four_combined-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_four_combined-best-head-evaluation/0/report.json new file mode 100644 index 000000000..02eadaaea --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_four_combined-best-head-evaluation/0/report.json @@ -0,0 +1,445 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.887499988079071, + "min_features_per_member": 92.0, + "mean_features_per_member": 97.625, + "max_features_per_member": 103.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.399999618530273, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.797659951359852, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_four_combined-evaluation/_esam_diagnostics/all_four_combined-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.788873839009288, + "recall": 0.9686263795008256, + "f1-score": 0.8695577986445486, + "support": 84179.0 + }, + "1": { + "precision": 0.5584350443069721, + "recall": 0.13273984579922105, + "f1-score": 0.21449442892463796, + "support": 25162.0 + }, + "accuracy": 0.7762687372531805, + "macro avg": { + "precision": 0.67365444165813, + "recall": 0.5506831126500233, + "f1-score": 0.5420261137845933, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7358443171254596, + "recall": 0.7762687372531805, + "f1-score": 0.7188119255604138, + "support": 109341.0 + }, + "cross-entropy": 0.4989015904618268, + "roc-auc": 0.6823993201798577, + "score": 0.6823993201798577 + }, + "val": { + "0": { + "precision": 0.6940704108113401, + "recall": 0.9134553421832711, + "f1-score": 0.7887927436632258, + "support": 15518.0 + }, + "1": { + "precision": 0.6500781657113079, + "recall": 0.28537115406611, + "f1-score": 0.39662983864557666, + "support": 8743.0 + }, + "accuracy": 0.6871110011953341, + "macro avg": { + "precision": 0.672074288261324, + "recall": 0.5994132481246905, + "f1-score": 0.5927112911544012, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.678216810427614, + "recall": 0.6871110011953341, + "f1-score": 0.6474679722783157, + "support": 24261.0 + }, + "cross-entropy": 0.6244923069797662, + "roc-auc": 0.6479868261151002, + "score": 0.6479868261151002 + }, + "test": { + "0": { + "precision": 0.6674781317049018, + "recall": 0.9559744711027065, + "f1-score": 0.7860926186889547, + "support": 16922.0 + }, + "1": { + "precision": 0.6642631816133393, + "recall": 0.15462079093674605, + "f1-score": 0.25085091899251194, + "support": 9533.0 + }, + "accuracy": 0.6672084672084672, + "macro avg": { + "precision": 0.6658706566591206, + "recall": 0.5552976310197263, + "f1-score": 0.5184717688407333, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6663196316397774, + "recall": 0.6672084672084672, + "f1-score": 0.593219470958613, + "support": 26455.0 + }, + "cross-entropy": 0.65875352245754, + "roc-auc": 0.6047175337399694, + "score": 0.6047175337399694 + } + }, + "time": "0:03:40.582211", + "diversity": { + "train": { + "mean_centered_corr": -0.010312517173588276, + "mean_pairwise_disagreement": 0.02666120948639223, + "member_std": 0.029578423127532005 + }, + "val": { + "mean_centered_corr": -0.012083624489605427, + "mean_pairwise_disagreement": 0.09328301030061521, + "member_std": 0.06631172448396683 + }, + "test": { + "mean_centered_corr": -0.010351695120334625, + "mean_pairwise_disagreement": 0.058351141012431336, + "member_std": 0.06402193009853363 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 22 + ], + "head_selection_time": "0:00:04.820494" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_four_combined-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_four_combined-evaluation/0.toml new file mode 100644 index 000000000..b8da71126 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_four_combined-evaluation/0.toml @@ -0,0 +1,74 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 47 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "ecom-offers" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_four_combined-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_four_combined-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_four_combined-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_four_combined-evaluation/0/report.json new file mode 100644 index 000000000..81bd76de1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_four_combined-evaluation/0/report.json @@ -0,0 +1,441 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.887499988079071, + "min_features_per_member": 92.0, + "mean_features_per_member": 97.625, + "max_features_per_member": 103.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.399999618530273, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.797659951359852, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_four_combined-evaluation/_esam_diagnostics/all_four_combined-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7843377382851067, + "recall": 0.9814680621057509, + "f1-score": 0.8718993219534074, + "support": 84179.0 + }, + "1": { + "precision": 0.6104868913857678, + "recall": 0.09717033622128607, + "f1-score": 0.16765522679740802, + "support": 25162.0 + }, + "accuracy": 0.7779698374809083, + "macro avg": { + "precision": 0.6974123148354372, + "recall": 0.5393191991635184, + "f1-score": 0.5197772743754077, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7443304673649471, + "recall": 0.7779698374809083, + "f1-score": 0.7098357783392529, + "support": 109341.0 + }, + "cross-entropy": 0.49273880116728924, + "roc-auc": 0.6982248815909875, + "score": 0.6982248815909875 + }, + "val": { + "0": { + "precision": 0.6767916685698625, + "recall": 0.9548266529191907, + "f1-score": 0.7921199647162599, + "support": 15518.0 + }, + "1": { + "precision": 0.7039695945945946, + "recall": 0.19066681916962142, + "f1-score": 0.3000630006300063, + "support": 8743.0 + }, + "accuracy": 0.6794443757470838, + "macro avg": { + "precision": 0.6903806315822285, + "recall": 0.5727467360444061, + "f1-score": 0.5460914826731331, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6865858488276521, + "recall": 0.6794443757470838, + "f1-score": 0.6147961100933624, + "support": 24261.0 + }, + "cross-entropy": 0.6764607634104041, + "roc-auc": 0.6244437709503304, + "score": 0.6244437709503304 + }, + "test": { + "0": { + "precision": 0.657613234183004, + "recall": 0.9772485521805933, + "f1-score": 0.7861846014880316, + "support": 16922.0 + }, + "1": { + "precision": 0.7056574923547401, + "recall": 0.09682156718766391, + "f1-score": 0.17027949451157642, + "support": 9533.0 + }, + "accuracy": 0.65998865998866, + "macro avg": { + "precision": 0.6816353632688721, + "recall": 0.5370350596841286, + "f1-score": 0.478232047999804, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6749258750127587, + "recall": 0.65998865998866, + "f1-score": 0.5642445756023182, + "support": 26455.0 + }, + "cross-entropy": 0.7397410477591574, + "roc-auc": 0.592244305956134, + "score": 0.592244305956134 + } + }, + "time": "0:03:40.582211", + "diversity": { + "train": { + "mean_centered_corr": -0.010312517173588276, + "mean_pairwise_disagreement": 0.02666120948639223, + "member_std": 0.029578423127532005 + }, + "val": { + "mean_centered_corr": -0.012083624489605427, + "mean_pairwise_disagreement": 0.09328301030061521, + "member_std": 0.06631172448396683 + }, + "test": { + "mean_centered_corr": -0.010351695120334625, + "mean_pairwise_disagreement": 0.058351141012431336, + "member_std": 0.06402193009853363 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_four_combined-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_four_combined-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_four_combined-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_four_combined-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..9e8350c7b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_four_combined-greedy-heads-evaluation/0/report.json @@ -0,0 +1,447 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.887499988079071, + "min_features_per_member": 92.0, + "mean_features_per_member": 97.625, + "max_features_per_member": 103.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.399999618530273, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.797659951359852, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_four_combined-evaluation/_esam_diagnostics/all_four_combined-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7909716267830696, + "recall": 0.9643616578956747, + "f1-score": 0.8691029971468489, + "support": 84179.0 + }, + "1": { + "precision": 0.5528394693695037, + "recall": 0.1474048167872188, + "f1-score": 0.2327507765680399, + "support": 25162.0 + }, + "accuracy": 0.7763601942546712, + "macro avg": { + "precision": 0.6719055480762867, + "recall": 0.5558832373414467, + "f1-score": 0.5509268868574444, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.736171676674326, + "recall": 0.7763601942546712, + "f1-score": 0.7226630105525798, + "support": 109341.0 + }, + "cross-entropy": 0.5000760433868655, + "roc-auc": 0.6809180604528166, + "score": 0.6809180604528166 + }, + "val": { + "0": { + "precision": 0.7073092712054166, + "recall": 0.8549426472483568, + "f1-score": 0.7741502552881109, + "support": 15518.0 + }, + "1": { + "precision": 0.5910247093023255, + "recall": 0.37206908383849935, + "f1-score": 0.45665754193865377, + "support": 8743.0 + }, + "accuracy": 0.6809282387370678, + "macro avg": { + "precision": 0.649166990253871, + "recall": 0.613505865543428, + "f1-score": 0.6154038986133823, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6654034996082555, + "recall": 0.6809282387370678, + "f1-score": 0.6597345760986997, + "support": 24261.0 + }, + "cross-entropy": 0.6204112258943162, + "roc-auc": 0.6505096994576862, + "score": 0.6505096994576862 + }, + "test": { + "0": { + "precision": 0.6799947561615102, + "recall": 0.9195721545916559, + "f1-score": 0.7818419333768779, + "support": 16922.0 + }, + "1": { + "precision": 0.6188742649117894, + "recall": 0.2318262876324347, + "f1-score": 0.3373015873015873, + "support": 9533.0 + }, + "accuracy": 0.6717444717444717, + "macro avg": { + "precision": 0.6494345105366498, + "recall": 0.5756992211120453, + "f1-score": 0.5595717603392326, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6579701240283184, + "recall": 0.6717444717444717, + "f1-score": 0.6216528152844286, + "support": 26455.0 + }, + "cross-entropy": 0.6441552092220911, + "roc-auc": 0.6063775930816054, + "score": 0.6063775930816054 + } + }, + "time": "0:03:40.582211", + "diversity": { + "train": { + "mean_centered_corr": -0.010312517173588276, + "mean_pairwise_disagreement": 0.02666120948639223, + "member_std": 0.029578423127532005 + }, + "val": { + "mean_centered_corr": -0.012083624489605427, + "mean_pairwise_disagreement": 0.09328301030061521, + "member_std": 0.06631172448396683 + }, + "test": { + "mean_centered_corr": -0.010351695120334625, + "mean_pairwise_disagreement": 0.058351141012431336, + "member_std": 0.06402193009853363 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 22, + 24, + 24 + ], + "head_selection_time": "0:00:20.539821" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_cf_fisd-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_cf_fisd-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_cf_fisd-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_cf_fisd-best-head-evaluation/0/report.json new file mode 100644 index 000000000..fb5a78bba --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_cf_fisd-best-head-evaluation/0/report.json @@ -0,0 +1,274 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.887499988079071, + "min_features_per_member": 92.0, + "mean_features_per_member": 97.625, + "max_features_per_member": 103.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.399999618530273, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.797659951359852, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_cf_fisd-evaluation/_esam_diagnostics/all_minus_cf_fisd-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.791864764923402, + "recall": 0.9615937466589054, + "f1-score": 0.8685146538913417, + "support": 84179.0 + }, + "1": { + "precision": 0.5458631830313246, + "recall": 0.1544392337652015, + "f1-score": 0.24076081905765, + "support": 25162.0 + }, + "accuracy": 0.7758480350463229, + "macro avg": { + "precision": 0.6688639739773633, + "recall": 0.5580164902120535, + "f1-score": 0.5546377364744959, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7352538705327485, + "recall": 0.7758480350463229, + "f1-score": 0.7240533631396078, + "support": 109341.0 + }, + "cross-entropy": 0.5019628647078745, + "roc-auc": 0.677748911462424, + "score": 0.677748911462424 + }, + "val": { + "0": { + "precision": 0.709374145941514, + "recall": 0.8363191132877948, + "f1-score": 0.7676337503327123, + "support": 15518.0 + }, + "1": { + "precision": 0.5742541066040898, + "recall": 0.39185634221663046, + "f1-score": 0.465837242504589, + "support": 8743.0 + }, + "accuracy": 0.6761469024360084, + "macro avg": { + "precision": 0.6418141262728019, + "recall": 0.6140877277522127, + "f1-score": 0.6167354964186507, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.660680584096285, + "recall": 0.6761469024360084, + "f1-score": 0.6588745537645048, + "support": 24261.0 + }, + "cross-entropy": 0.6230257230038371, + "roc-auc": 0.6498655113216565, + "score": 0.6498655113216565 + }, + "test": { + "0": { + "precision": 0.6823990382474732, + "recall": 0.9056849072213686, + "f1-score": 0.7783448871283106, + "support": 16922.0 + }, + "1": { + "precision": 0.6006006006006006, + "recall": 0.25175705444246305, + "f1-score": 0.3547934067558578, + "support": 9533.0 + }, + "accuracy": 0.6700434700434701, + "macro avg": { + "precision": 0.6414998194240369, + "recall": 0.5787209808319158, + "f1-score": 0.5665691469420842, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6529231544414767, + "recall": 0.6700434700434701, + "f1-score": 0.6257190597841189, + "support": 26455.0 + }, + "cross-entropy": 0.6443544898598349, + "roc-auc": 0.6047694345185003, + "score": 0.6047694345185003 + } + }, + "time": "0:03:18.117749", + "diversity": { + "train": { + "mean_centered_corr": -0.011164821684360504, + "mean_pairwise_disagreement": 0.026747282325698445, + "member_std": 0.029834141954779625 + }, + "val": { + "mean_centered_corr": -0.007544255815446377, + "mean_pairwise_disagreement": 0.09209897804919884, + "member_std": 0.06599097698926926 + }, + "test": { + "mean_centered_corr": -0.005217556841671467, + "mean_pairwise_disagreement": 0.057501402259466775, + "member_std": 0.06378328055143356 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 24 + ], + "head_selection_time": "0:00:04.771288" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_cf_fisd-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_cf_fisd-evaluation/0.toml new file mode 100644 index 000000000..972126362 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_cf_fisd-evaluation/0.toml @@ -0,0 +1,63 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_cf_fisd-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_cf_fisd-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_cf_fisd-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_cf_fisd-evaluation/0/report.json new file mode 100644 index 000000000..51c1a0f19 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_cf_fisd-evaluation/0/report.json @@ -0,0 +1,270 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.887499988079071, + "min_features_per_member": 92.0, + "mean_features_per_member": 97.625, + "max_features_per_member": 103.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.399999618530273, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.797659951359852, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_cf_fisd-evaluation/_esam_diagnostics/all_minus_cf_fisd-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7843520689556971, + "recall": 0.9815512182373276, + "f1-score": 0.8719409883812961, + "support": 84179.0 + }, + "1": { + "precision": 0.6115557778889444, + "recall": 0.09717033622128607, + "f1-score": 0.16769547325102882, + "support": 25162.0 + }, + "accuracy": 0.7780338573819519, + "macro avg": { + "precision": 0.6979539234223208, + "recall": 0.5393607772293069, + "f1-score": 0.5198182308161624, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7445874767549523, + "recall": 0.7780338573819519, + "f1-score": 0.7098771179968311, + "support": 109341.0 + }, + "cross-entropy": 0.4926987595087899, + "roc-auc": 0.69829445109446, + "score": 0.69829445109446 + }, + "val": { + "0": { + "precision": 0.6763216713041098, + "recall": 0.9554710658590024, + "f1-score": 0.7920194439251088, + "support": 15518.0 + }, + "1": { + "precision": 0.704448246364414, + "recall": 0.18837927484845018, + "f1-score": 0.29726558974821765, + "support": 8743.0 + }, + "accuracy": 0.6790321915831994, + "macro avg": { + "precision": 0.6903849588342619, + "recall": 0.5719251703537263, + "f1-score": 0.5446425168366632, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6864577186950763, + "recall": 0.6790321915831994, + "f1-score": 0.6137237039692719, + "support": 24261.0 + }, + "cross-entropy": 0.6773915501327039, + "roc-auc": 0.6244116424360375, + "score": 0.6244116424360375 + }, + "test": { + "0": { + "precision": 0.6572393247269116, + "recall": 0.9777804042075405, + "f1-score": 0.7860892700192412, + "support": 16922.0 + }, + "1": { + "precision": 0.70625, + "recall": 0.09482849050666108, + "f1-score": 0.1672061407564968, + "support": 9533.0 + }, + "accuracy": 0.6596106596106596, + "macro avg": { + "precision": 0.6817446623634558, + "recall": 0.5363044473571008, + "f1-score": 0.47664770538786905, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6749002117947003, + "recall": 0.6596106596106596, + "f1-score": 0.5630761204723979, + "support": 26455.0 + }, + "cross-entropy": 0.7412929728578691, + "roc-auc": 0.5923170352346188, + "score": 0.5923170352346188 + } + }, + "time": "0:03:18.117749", + "diversity": { + "train": { + "mean_centered_corr": -0.011164821684360504, + "mean_pairwise_disagreement": 0.026747282325698445, + "member_std": 0.029834141954779625 + }, + "val": { + "mean_centered_corr": -0.007544255815446377, + "mean_pairwise_disagreement": 0.09209897804919884, + "member_std": 0.06599097698926926 + }, + "test": { + "mean_centered_corr": -0.005217556841671467, + "mean_pairwise_disagreement": 0.057501402259466775, + "member_std": 0.06378328055143356 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_cf_fisd-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_cf_fisd-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_cf_fisd-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_cf_fisd-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..e7c657dd9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_cf_fisd-greedy-heads-evaluation/0/report.json @@ -0,0 +1,275 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.887499988079071, + "min_features_per_member": 92.0, + "mean_features_per_member": 97.625, + "max_features_per_member": 103.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.399999618530273, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.797659951359852, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_cf_fisd-evaluation/_esam_diagnostics/all_minus_cf_fisd-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7903111430597515, + "recall": 0.966476199527198, + "f1-score": 0.869561035046654, + "support": 84179.0 + }, + "1": { + "precision": 0.5589246639574867, + "recall": 0.14211906843653127, + "f1-score": 0.22661596958174904, + "support": 25162.0 + }, + "accuracy": 0.7767717507613795, + "macro avg": { + "precision": 0.674617903508619, + "recall": 0.5542976339818646, + "f1-score": 0.5480885023142016, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.737063536149524, + "recall": 0.7767717507613795, + "f1-score": 0.7216038759093868, + "support": 109341.0 + }, + "cross-entropy": 0.49896313282151494, + "roc-auc": 0.6832990228404343, + "score": 0.6832990228404343 + }, + "val": { + "0": { + "precision": 0.7028659160696008, + "recall": 0.8850367315375692, + "f1-score": 0.7835016258771179, + "support": 15518.0 + }, + "1": { + "precision": 0.6221139589070113, + "recall": 0.33592588356399405, + "f1-score": 0.4362745098039216, + "support": 8743.0 + }, + "accuracy": 0.6871522196117226, + "macro avg": { + "precision": 0.662489937488306, + "recall": 0.6104813075507817, + "f1-score": 0.6098880678405197, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6737651221422063, + "recall": 0.6871522196117226, + "f1-score": 0.6583704822380281, + "support": 24261.0 + }, + "cross-entropy": 0.6202643229081582, + "roc-auc": 0.6505072081895442, + "score": 0.6505072081895442 + }, + "test": { + "0": { + "precision": 0.6762482381582882, + "recall": 0.9356459047393925, + "f1-score": 0.7850749969009545, + "support": 16922.0 + }, + "1": { + "precision": 0.6420118343195266, + "recall": 0.2048673030525543, + "f1-score": 0.31061630218687875, + "support": 9533.0 + }, + "accuracy": 0.6723114723114723, + "macro avg": { + "precision": 0.6591300362389074, + "recall": 0.5702566038959733, + "f1-score": 0.5478456495439166, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6639112267126289, + "recall": 0.6723114723114723, + "f1-score": 0.6141048688832155, + "support": 26455.0 + }, + "cross-entropy": 0.6493383320901536, + "roc-auc": 0.6062618182365493, + "score": 0.6062618182365493 + } + }, + "time": "0:03:18.117749", + "diversity": { + "train": { + "mean_centered_corr": -0.011164821684360504, + "mean_pairwise_disagreement": 0.026747282325698445, + "member_std": 0.029834141954779625 + }, + "val": { + "mean_centered_corr": -0.007544255815446377, + "mean_pairwise_disagreement": 0.09209897804919884, + "member_std": 0.06599097698926926 + }, + "test": { + "mean_centered_corr": -0.005217556841671467, + "mean_pairwise_disagreement": 0.057501402259466775, + "member_std": 0.06378328055143356 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 24, + 22 + ], + "head_selection_time": "0:00:15.900575" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_esam-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_esam-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_esam-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_esam-best-head-evaluation/0/report.json new file mode 100644 index 000000000..1712e7390 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_esam-best-head-evaluation/0/report.json @@ -0,0 +1,435 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.887499988079071, + "min_features_per_member": 92.0, + "mean_features_per_member": 97.625, + "max_features_per_member": 103.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.399999618530273, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.797659951359852, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7914072107595307, + "recall": 0.9632568692904406, + "f1-score": 0.8689166671131662, + "support": 84179.0 + }, + "1": { + "precision": 0.5506319918640128, + "recall": 0.15062395676019394, + "f1-score": 0.2365423623030114, + "support": 25162.0 + }, + "accuracy": 0.7762504458528823, + "macro avg": { + "precision": 0.6710196013117717, + "recall": 0.5569404130253173, + "f1-score": 0.5527295147080888, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7359990284871075, + "recall": 0.7762504458528823, + "f1-score": 0.7233920948334804, + "support": 109341.0 + }, + "cross-entropy": 0.5022261934082275, + "roc-auc": 0.6755481612167327, + "score": 0.6755481612167327 + }, + "val": { + "0": { + "precision": 0.7100723427935448, + "recall": 0.8222709111998969, + "f1-score": 0.7620640229335881, + "support": 15518.0 + }, + "1": { + "precision": 0.5615959306946431, + "recall": 0.40409470433489647, + "f1-score": 0.470001330317946, + "support": 8743.0 + }, + "accuracy": 0.6715716582168914, + "macro avg": { + "precision": 0.635834136744094, + "recall": 0.6131828077673966, + "f1-score": 0.6160326766257671, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6565655099762373, + "recall": 0.6715716582168914, + "f1-score": 0.656812626802408, + "support": 24261.0 + }, + "cross-entropy": 0.6242117770764835, + "roc-auc": 0.6476721782117019, + "score": 0.6476721782117019 + }, + "test": { + "0": { + "precision": 0.6831312812654919, + "recall": 0.8957570027183548, + "f1-score": 0.775127202065915, + "support": 16922.0 + }, + "1": { + "precision": 0.5864978902953587, + "recall": 0.26245672925626773, + "f1-score": 0.36263497354880786, + "support": 9533.0 + }, + "accuracy": 0.6675486675486676, + "macro avg": { + "precision": 0.6348145857804253, + "recall": 0.5791068659873113, + "f1-score": 0.5688810878073615, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6483096552545949, + "recall": 0.6675486675486676, + "f1-score": 0.626486551358919, + "support": 26455.0 + }, + "cross-entropy": 0.6431486794584158, + "roc-auc": 0.6040083543113315, + "score": 0.6040083543113315 + } + }, + "time": "0:02:18.381792", + "diversity": { + "train": { + "mean_centered_corr": -0.010318981483578682, + "mean_pairwise_disagreement": 0.026643028719563627, + "member_std": 0.029531365260481834 + }, + "val": { + "mean_centered_corr": -0.01181983295828104, + "mean_pairwise_disagreement": 0.09317281751809289, + "member_std": 0.06609699130058289 + }, + "test": { + "mean_centered_corr": -0.010081159882247448, + "mean_pairwise_disagreement": 0.05819193883710013, + "member_std": 0.06373059004545212 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 24 + ], + "head_selection_time": "0:00:04.830833" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_esam-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_esam-evaluation/0.toml new file mode 100644 index 000000000..772f87508 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_esam-evaluation/0.toml @@ -0,0 +1,64 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 47 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "ecom-offers" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_esam-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_esam-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_esam-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_esam-evaluation/0/report.json new file mode 100644 index 000000000..5af8eb5f7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_esam-evaluation/0/report.json @@ -0,0 +1,431 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.887499988079071, + "min_features_per_member": 92.0, + "mean_features_per_member": 97.625, + "max_features_per_member": 103.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.399999618530273, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.797659951359852, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7843261973702947, + "recall": 0.9814443032110146, + "f1-score": 0.8718828160452526, + "support": 84179.0 + }, + "1": { + "precision": 0.6100848726909636, + "recall": 0.09713059375248391, + "f1-score": 0.1675809105869446, + "support": 25162.0 + }, + "accuracy": 0.7779424003804611, + "macro avg": { + "precision": 0.6972055350306292, + "recall": 0.5392874484817493, + "f1-score": 0.5197318633160987, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7442290680996522, + "recall": 0.7779424003804611, + "f1-score": 0.7098059688868954, + "support": 109341.0 + }, + "cross-entropy": 0.49269774798365623, + "roc-auc": 0.6983158737104703, + "score": 0.6983158737104703 + }, + "val": { + "0": { + "precision": 0.6767916685698625, + "recall": 0.9548266529191907, + "f1-score": 0.7921199647162599, + "support": 15518.0 + }, + "1": { + "precision": 0.7039695945945946, + "recall": 0.19066681916962142, + "f1-score": 0.3000630006300063, + "support": 8743.0 + }, + "accuracy": 0.6794443757470838, + "macro avg": { + "precision": 0.6903806315822285, + "recall": 0.5727467360444061, + "f1-score": 0.5460914826731331, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6865858488276521, + "recall": 0.6794443757470838, + "f1-score": 0.6147961100933624, + "support": 24261.0 + }, + "cross-entropy": 0.6766578943018672, + "roc-auc": 0.6245232151327823, + "score": 0.6245232151327823 + }, + "test": { + "0": { + "precision": 0.6575609368165732, + "recall": 0.9772485521805933, + "f1-score": 0.7861472273062204, + "support": 16922.0 + }, + "1": { + "precision": 0.7052067381316999, + "recall": 0.09661176964229519, + "f1-score": 0.16994187655687795, + "support": 9533.0 + }, + "accuracy": 0.6599130599130599, + "macro avg": { + "precision": 0.6813838374741366, + "recall": 0.5369301609114442, + "f1-score": 0.4780445519315492, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6747299946104535, + "recall": 0.6599130599130599, + "f1-score": 0.5640990092493887, + "support": 26455.0 + }, + "cross-entropy": 0.7401458486738616, + "roc-auc": 0.5923207050179439, + "score": 0.5923207050179439 + } + }, + "time": "0:02:18.381792", + "diversity": { + "train": { + "mean_centered_corr": -0.010318981483578682, + "mean_pairwise_disagreement": 0.026643028719563627, + "member_std": 0.029531365260481834 + }, + "val": { + "mean_centered_corr": -0.01181983295828104, + "mean_pairwise_disagreement": 0.09317281751809289, + "member_std": 0.06609699130058289 + }, + "test": { + "mean_centered_corr": -0.010081159882247448, + "mean_pairwise_disagreement": 0.05819193883710013, + "member_std": 0.06373059004545212 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_esam-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_esam-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_esam-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_esam-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..1cc02fd88 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_esam-greedy-heads-evaluation/0/report.json @@ -0,0 +1,436 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.887499988079071, + "min_features_per_member": 92.0, + "mean_features_per_member": 97.625, + "max_features_per_member": 103.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.399999618530273, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.797659951359852, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.790500918983575, + "recall": 0.9656565176587986, + "f1-score": 0.8693438853537244, + "support": 84179.0 + }, + "1": { + "precision": 0.5559139784946237, + "recall": 0.14382799459502424, + "f1-score": 0.22852993180095985, + "support": 25162.0 + }, + "accuracy": 0.7765339625575036, + "macro avg": { + "precision": 0.6732074487390993, + "recall": 0.5547422561269114, + "f1-score": 0.5489369085773421, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.736516808754265, + "recall": 0.7765339625575036, + "f1-score": 0.7218771464424774, + "support": 109341.0 + }, + "cross-entropy": 0.4993286461130596, + "roc-auc": 0.6825283693048605, + "score": 0.6825283693048605 + }, + "val": { + "0": { + "precision": 0.7052691926825455, + "recall": 0.8720195901533703, + "f1-score": 0.7798299956778563, + "support": 15518.0 + }, + "1": { + "precision": 0.6085928261726449, + "recall": 0.35319684318883676, + "f1-score": 0.4469855974524137, + "support": 8743.0 + }, + "accuracy": 0.685050080375912, + "macro avg": { + "precision": 0.6569310094275952, + "recall": 0.6126082166711035, + "f1-score": 0.613407796565135, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6704296777245445, + "recall": 0.685050080375912, + "f1-score": 0.6598819896729494, + "support": 24261.0 + }, + "cross-entropy": 0.6199389736027974, + "roc-auc": 0.6504598446123828, + "score": 0.6504598446123828 + }, + "test": { + "0": { + "precision": 0.6781034482758621, + "recall": 0.929677343103652, + "f1-score": 0.7842081650964559, + "support": 16922.0 + }, + "1": { + "precision": 0.6344086021505376, + "recall": 0.21661596559320256, + "f1-score": 0.32295902408507976, + "support": 9533.0 + }, + "accuracy": 0.6727272727272727, + "macro avg": { + "precision": 0.6562560252131999, + "recall": 0.5731466543484273, + "f1-score": 0.5535835945907678, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6623581083358614, + "recall": 0.6727272727272727, + "f1-score": 0.6179980701706782, + "support": 26455.0 + }, + "cross-entropy": 0.6464247215340417, + "roc-auc": 0.6064818006704371, + "score": 0.6064818006704371 + } + }, + "time": "0:02:18.381792", + "diversity": { + "train": { + "mean_centered_corr": -0.010318981483578682, + "mean_pairwise_disagreement": 0.026643028719563627, + "member_std": 0.029531365260481834 + }, + "val": { + "mean_centered_corr": -0.01181983295828104, + "mean_pairwise_disagreement": 0.09317281751809289, + "member_std": 0.06609699130058289 + }, + "test": { + "mean_centered_corr": -0.010081159882247448, + "mean_pairwise_disagreement": 0.05819193883710013, + "member_std": 0.06373059004545212 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 24, + 22 + ], + "head_selection_time": "0:00:15.884778" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_mfb-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_mfb-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_mfb-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_mfb-best-head-evaluation/0/report.json new file mode 100644 index 000000000..ba2bc2674 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_mfb-best-head-evaluation/0/report.json @@ -0,0 +1,409 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_mfb-evaluation/_esam_diagnostics/all_minus_mfb-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7819656172915348, + "recall": 0.9828935957899239, + "f1-score": 0.8709918521169758, + "support": 84179.0 + }, + "1": { + "precision": 0.5922989807474519, + "recall": 0.08314124473412289, + "f1-score": 0.14581445598382936, + "support": 25162.0 + }, + "accuracy": 0.7758388893461738, + "macro avg": { + "precision": 0.6871322990194934, + "recall": 0.5330174202620234, + "f1-score": 0.5084031540504026, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7383187519004901, + "recall": 0.7758388893461738, + "f1-score": 0.7041110513057317, + "support": 109341.0 + }, + "cross-entropy": 0.500751581030381, + "roc-auc": 0.6780775151909603, + "score": 0.6780775151909603 + }, + "val": { + "0": { + "precision": 0.6903867726674145, + "recall": 0.9121665163036474, + "f1-score": 0.7859304294716971, + "support": 15518.0 + }, + "1": { + "precision": 0.6373070782331027, + "recall": 0.2739334324602539, + "f1-score": 0.38316934645228384, + "support": 8743.0 + }, + "accuracy": 0.682164791228721, + "macro avg": { + "precision": 0.6638469254502586, + "recall": 0.5930499743819506, + "f1-score": 0.5845498879619905, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6712583044081017, + "recall": 0.682164791228721, + "f1-score": 0.6407863649715229, + "support": 24261.0 + }, + "cross-entropy": 0.6287444440087023, + "roc-auc": 0.637709740638791, + "score": 0.637709740638791 + }, + "test": { + "0": { + "precision": 0.6690629440775725, + "recall": 0.945987471930032, + "f1-score": 0.7837837837837838, + "support": 16922.0 + }, + "1": { + "precision": 0.638592328983788, + "recall": 0.16941151788524075, + "f1-score": 0.2677831205438567, + "support": 9533.0 + }, + "accuracy": 0.6661500661500661, + "macro avg": { + "precision": 0.6538276365306803, + "recall": 0.5576994949076364, + "f1-score": 0.5257834521638203, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6580829261721086, + "recall": 0.6661500661500661, + "f1-score": 0.5978441004473171, + "support": 26455.0 + }, + "cross-entropy": 0.6564068211877354, + "roc-auc": 0.6004564813723224, + "score": 0.6004564813723224 + } + }, + "time": "0:03:16.099007", + "diversity": { + "train": { + "mean_centered_corr": -0.01766625978052616, + "mean_pairwise_disagreement": 0.024688098434875683, + "member_std": 0.027821816504001617 + }, + "val": { + "mean_centered_corr": -0.004993404261767864, + "mean_pairwise_disagreement": 0.06782124769476035, + "member_std": 0.05843981355428696 + }, + "test": { + "mean_centered_corr": 0.0002984397578984499, + "mean_pairwise_disagreement": 0.04484334323044, + "member_std": 0.059851765632629395 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 14 + ], + "head_selection_time": "0:00:04.874514" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_mfb-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_mfb-evaluation/0.toml new file mode 100644 index 000000000..8d636ee4c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_mfb-evaluation/0.toml @@ -0,0 +1,61 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "ecom-offers" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_mfb-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_mfb-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_mfb-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_mfb-evaluation/0/report.json new file mode 100644 index 000000000..5477da392 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_mfb-evaluation/0/report.json @@ -0,0 +1,405 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_mfb-evaluation/_esam_diagnostics/all_minus_mfb-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7813800636546827, + "recall": 0.9857684220530061, + "f1-score": 0.8717544661382415, + "support": 84179.0 + }, + "1": { + "precision": 0.6188355074769328, + "recall": 0.07729910182020507, + "f1-score": 0.13743154919625508, + "support": 25162.0 + }, + "accuracy": 0.7767077308603361, + "macro avg": { + "precision": 0.7001077855658078, + "recall": 0.5315337619366056, + "f1-score": 0.5045930076672482, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7439746427920186, + "recall": 0.7767077308603361, + "f1-score": 0.7027690605164321, + "support": 109341.0 + }, + "cross-entropy": 0.4957520792555438, + "roc-auc": 0.6913928653833158, + "score": 0.6913928653833158 + }, + "val": { + "0": { + "precision": 0.6750442317288935, + "recall": 0.9588864544400052, + "f1-score": 0.7923111738239131, + "support": 15518.0 + }, + "1": { + "precision": 0.7123534715960325, + "recall": 0.18071600137252658, + "f1-score": 0.2882948636073351, + "support": 8743.0 + }, + "accuracy": 0.6784551337537612, + "macro avg": { + "precision": 0.6936988516624629, + "recall": 0.5698012279062659, + "f1-score": 0.5403030187156241, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6884894600442307, + "recall": 0.6784551337537612, + "f1-score": 0.6106774983685097, + "support": 24261.0 + }, + "cross-entropy": 0.6741937219356421, + "roc-auc": 0.6226455728683622, + "score": 0.6226455728683622 + }, + "test": { + "0": { + "precision": 0.6571859256614702, + "recall": 0.9790213922704172, + "f1-score": 0.7864517813486507, + "support": 16922.0 + }, + "1": { + "precision": 0.7150882825040128, + "recall": 0.0934648064617644, + "f1-score": 0.16532145839131646, + "support": 9533.0 + }, + "accuracy": 0.6599130599130599, + "macro avg": { + "precision": 0.6861371040827415, + "recall": 0.5362430993660908, + "f1-score": 0.4758866198699836, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6780509102685373, + "recall": 0.6599130599130599, + "f1-score": 0.5626288605868942, + "support": 26455.0 + }, + "cross-entropy": 0.7346957668444309, + "roc-auc": 0.5910976319446108, + "score": 0.5910976319446108 + } + }, + "time": "0:03:16.099007", + "diversity": { + "train": { + "mean_centered_corr": -0.01766625978052616, + "mean_pairwise_disagreement": 0.024688098434875683, + "member_std": 0.027821816504001617 + }, + "val": { + "mean_centered_corr": -0.004993404261767864, + "mean_pairwise_disagreement": 0.06782124769476035, + "member_std": 0.05843981355428696 + }, + "test": { + "mean_centered_corr": 0.0002984397578984499, + "mean_pairwise_disagreement": 0.04484334323044, + "member_std": 0.059851765632629395 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_mfb-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_mfb-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_mfb-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_mfb-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..3a857ba75 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_mfb-greedy-heads-evaluation/0/report.json @@ -0,0 +1,411 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_mfb-evaluation/_esam_diagnostics/all_minus_mfb-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7814865438092096, + "recall": 0.9848537046056618, + "f1-score": 0.8714628095698608, + "support": 84179.0 + }, + "1": { + "precision": 0.6084152334152334, + "recall": 0.0787298306970829, + "f1-score": 0.13941867830248433, + "support": 25162.0 + }, + "accuracy": 0.7763327571542239, + "macro avg": { + "precision": 0.6949508886122215, + "recall": 0.5317917676513724, + "f1-score": 0.5054407439361726, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7416586630313383, + "recall": 0.7763327571542239, + "f1-score": 0.7030018074668096, + "support": 109341.0 + }, + "cross-entropy": 0.49890429497744304, + "roc-auc": 0.6841500123073285, + "score": 0.6841500123073285 + }, + "val": { + "0": { + "precision": 0.6818666295935863, + "recall": 0.9481891996391287, + "f1-score": 0.7932716931288244, + "support": 15518.0 + }, + "1": { + "precision": 0.7002237136465325, + "recall": 0.21480041175797782, + "f1-score": 0.3287527352297593, + "support": 8743.0 + }, + "accuracy": 0.6838959647170356, + "macro avg": { + "precision": 0.6910451716200594, + "recall": 0.5814948056985533, + "f1-score": 0.5610122141792918, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6884820199680519, + "recall": 0.6838959647170356, + "f1-score": 0.6258717817932848, + "support": 24261.0 + }, + "cross-entropy": 0.6354781186535776, + "roc-auc": 0.6386704340734016, + "score": 0.6386704340734016 + }, + "test": { + "0": { + "precision": 0.663352330140253, + "recall": 0.9698617184729937, + "f1-score": 0.7878453303890742, + "support": 16922.0 + }, + "1": { + "precision": 0.7024504084014003, + "recall": 0.12629812231196896, + "f1-score": 0.21410153818796124, + "support": 9533.0 + }, + "accuracy": 0.6658854658854659, + "macro avg": { + "precision": 0.6829013692708266, + "recall": 0.5480799203924813, + "f1-score": 0.5009734342885177, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6774412350755588, + "recall": 0.6658854658854659, + "f1-score": 0.5810980398559723, + "support": 26455.0 + }, + "cross-entropy": 0.6721808608962874, + "roc-auc": 0.6021744792778928, + "score": 0.6021744792778928 + } + }, + "time": "0:03:16.099007", + "diversity": { + "train": { + "mean_centered_corr": -0.01766625978052616, + "mean_pairwise_disagreement": 0.024688098434875683, + "member_std": 0.027821816504001617 + }, + "val": { + "mean_centered_corr": -0.004993404261767864, + "mean_pairwise_disagreement": 0.06782124769476035, + "member_std": 0.05843981355428696 + }, + "test": { + "mean_centered_corr": 0.0002984397578984499, + "mean_pairwise_disagreement": 0.04484334323044, + "member_std": 0.059851765632629395 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 14, + 17, + 11 + ], + "head_selection_time": "0:00:20.397531" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_rla-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_rla-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_rla-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_rla-best-head-evaluation/0/report.json new file mode 100644 index 000000000..96a552803 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_rla-best-head-evaluation/0/report.json @@ -0,0 +1,436 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.887499988079071, + "min_features_per_member": 92.0, + "mean_features_per_member": 97.625, + "max_features_per_member": 103.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.399999618530273, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.797659951359852, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_rla-evaluation/_esam_diagnostics/all_minus_rla-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7887423366275359, + "recall": 0.9689708834745008, + "f1-score": 0.8696166701316147, + "support": 84179.0 + }, + "1": { + "precision": 0.5593048759912266, + "recall": 0.131746284079167, + "f1-score": 0.2132587088680884, + "support": 25162.0 + }, + "accuracy": 0.7763053200537767, + "macro avg": { + "precision": 0.6740236063093812, + "recall": 0.5503585837768339, + "f1-score": 0.5414376894998516, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7359432458516072, + "recall": 0.7763053200537767, + "f1-score": 0.71857288032438, + "support": 109341.0 + }, + "cross-entropy": 0.4989063469933395, + "roc-auc": 0.6823896934462292, + "score": 0.6823896934462292 + }, + "val": { + "0": { + "precision": 0.6937777995014419, + "recall": 0.9146797267689135, + "f1-score": 0.7890596770158712, + "support": 15518.0 + }, + "1": { + "precision": 0.65176223040505, + "recall": 0.28342674139311447, + "f1-score": 0.39505779194898366, + "support": 8743.0 + }, + "accuracy": 0.687193438028111, + "macro avg": { + "precision": 0.672770014953246, + "recall": 0.599053234081014, + "f1-score": 0.5920587344824274, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6786365390171356, + "recall": 0.687193438028111, + "f1-score": 0.6470721875826329, + "support": 24261.0 + }, + "cross-entropy": 0.6253138063351895, + "roc-auc": 0.6475121989956593, + "score": 0.6475121989956593 + }, + "test": { + "0": { + "precision": 0.667395158962517, + "recall": 0.9564472284599929, + "f1-score": 0.786194836421927, + "support": 16922.0 + }, + "1": { + "precision": 0.6656079854809437, + "recall": 0.15388649952795552, + "f1-score": 0.24997869983811877, + "support": 9533.0 + }, + "accuracy": 0.6672462672462672, + "macro avg": { + "precision": 0.6665015722217303, + "recall": 0.5551668639939742, + "f1-score": 0.5180867681300229, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6667511550010792, + "recall": 0.6672462672462672, + "f1-score": 0.5929705525416229, + "support": 26455.0 + }, + "cross-entropy": 0.6598558732426132, + "roc-auc": 0.6043243524106316, + "score": 0.6043243524106316 + } + }, + "time": "0:02:05.402786", + "diversity": { + "train": { + "mean_centered_corr": -0.010739956051111221, + "mean_pairwise_disagreement": 0.02668459002628946, + "member_std": 0.029851045459508896 + }, + "val": { + "mean_centered_corr": -0.013114765286445618, + "mean_pairwise_disagreement": 0.09411593809791634, + "member_std": 0.06746508926153183 + }, + "test": { + "mean_centered_corr": -0.01157404389232397, + "mean_pairwise_disagreement": 0.059169786185915216, + "member_std": 0.06565776467323303 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 22 + ], + "head_selection_time": "0:00:04.819667" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_rla-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_rla-evaluation/0.toml new file mode 100644 index 000000000..0bbba4c1e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_rla-evaluation/0.toml @@ -0,0 +1,67 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 47 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "ecom-offers" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_rla-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_rla-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_rla-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_rla-evaluation/0/report.json new file mode 100644 index 000000000..4c4ff62c1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_rla-evaluation/0/report.json @@ -0,0 +1,432 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.887499988079071, + "min_features_per_member": 92.0, + "mean_features_per_member": 97.625, + "max_features_per_member": 103.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.399999618530273, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.797659951359852, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_rla-evaluation/_esam_diagnostics/all_minus_rla-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7842236626080685, + "recall": 0.9816700127110087, + "f1-score": 0.8719085000422048, + "support": 84179.0 + }, + "1": { + "precision": 0.6111391129032258, + "recall": 0.09637548684524283, + "f1-score": 0.16649502231376587, + "support": 25162.0 + }, + "accuracy": 0.7779424003804611, + "macro avg": { + "precision": 0.6976813877556471, + "recall": 0.5390227497781258, + "f1-score": 0.5192017611779853, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7443927351456048, + "recall": 0.7779424003804611, + "f1-score": 0.7095758533076498, + "support": 109341.0 + }, + "cross-entropy": 0.49281466418323555, + "roc-auc": 0.6980493802481167, + "score": 0.6980493802481167 + }, + "val": { + "0": { + "precision": 0.6769005847953217, + "recall": 0.9547622116252095, + "f1-score": 0.7921723787627654, + "support": 15518.0 + }, + "1": { + "precision": 0.7041719342604298, + "recall": 0.19112432803385565, + "f1-score": 0.3006477150053976, + "support": 8743.0 + }, + "accuracy": 0.6795680309962491, + "macro avg": { + "precision": 0.6905362595278757, + "recall": 0.5729432698295326, + "f1-score": 0.5464100468840815, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6867284323025737, + "recall": 0.6795680309962491, + "f1-score": 0.6150403506010792, + "support": 24261.0 + }, + "cross-entropy": 0.6737994423706947, + "roc-auc": 0.6249365260993432, + "score": 0.6249365260993432 + }, + "test": { + "0": { + "precision": 0.6576644658340626, + "recall": 0.9771303628412717, + "f1-score": 0.7861829593001141, + "support": 16922.0 + }, + "1": { + "precision": 0.7052551408987052, + "recall": 0.09713626350571698, + "f1-score": 0.1707541950949659, + "support": 9533.0 + }, + "accuracy": 0.66002646002646, + "macro avg": { + "precision": 0.6814598033663839, + "recall": 0.5371333131734943, + "f1-score": 0.47846857719754, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6748136590070446, + "recall": 0.66002646002646, + "f1-score": 0.564414582465199, + "support": 26455.0 + }, + "cross-entropy": 0.7355041534289266, + "roc-auc": 0.5925964470819166, + "score": 0.5925964470819166 + } + }, + "time": "0:02:05.402786", + "diversity": { + "train": { + "mean_centered_corr": -0.010739956051111221, + "mean_pairwise_disagreement": 0.02668459002628946, + "member_std": 0.029851045459508896 + }, + "val": { + "mean_centered_corr": -0.013114765286445618, + "mean_pairwise_disagreement": 0.09411593809791634, + "member_std": 0.06746508926153183 + }, + "test": { + "mean_centered_corr": -0.01157404389232397, + "mean_pairwise_disagreement": 0.059169786185915216, + "member_std": 0.06565776467323303 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_rla-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_rla-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_rla-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_rla-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..08d8e98f7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_rla-greedy-heads-evaluation/0/report.json @@ -0,0 +1,438 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.887499988079071, + "min_features_per_member": 92.0, + "mean_features_per_member": 97.625, + "max_features_per_member": 103.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.399999618530273, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.797659951359852, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_rla-evaluation/_esam_diagnostics/all_minus_rla-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7910060901339829, + "recall": 0.9643378990009385, + "f1-score": 0.8691141517312263, + "support": 84179.0 + }, + "1": { + "precision": 0.5530077427039904, + "recall": 0.14760352913122962, + "f1-score": 0.23301336344814605, + "support": 25162.0 + }, + "accuracy": 0.7763876313551183, + "macro avg": { + "precision": 0.6720069164189867, + "recall": 0.555970714066084, + "f1-score": 0.5510637575896862, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7362369329282369, + "recall": 0.7763876313551183, + "f1-score": 0.7227320257695206, + "support": 109341.0 + }, + "cross-entropy": 0.5000639027992034, + "roc-auc": 0.6809037269331402, + "score": 0.6809037269331402 + }, + "val": { + "0": { + "precision": 0.7070610178523847, + "recall": 0.855007088542338, + "f1-score": 0.7740279438788904, + "support": 15518.0 + }, + "1": { + "precision": 0.5906113537117904, + "recall": 0.37126844332608944, + "f1-score": 0.4559308940234567, + "support": 8743.0 + }, + "accuracy": 0.680680928238737, + "macro avg": { + "precision": 0.6488361857820876, + "recall": 0.6131377659342137, + "f1-score": 0.6149794189511736, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6650957479302374, + "recall": 0.680680928238737, + "f1-score": 0.6593944783627922, + "support": 24261.0 + }, + "cross-entropy": 0.620372456282908, + "roc-auc": 0.6504789566191646, + "score": 0.6504789566191646 + }, + "test": { + "0": { + "precision": 0.6800699300699301, + "recall": 0.9195130599219951, + "f1-score": 0.7818702577759912, + "support": 16922.0 + }, + "1": { + "precision": 0.619020979020979, + "recall": 0.23214098395048777, + "f1-score": 0.33765639304241685, + "support": 9533.0 + }, + "accuracy": 0.6718200718200719, + "macro avg": { + "precision": 0.6495454545454545, + "recall": 0.5758270219362414, + "f1-score": 0.559763325409204, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6580710772878605, + "recall": 0.6718200718200719, + "f1-score": 0.6217987865038247, + "support": 26455.0 + }, + "cross-entropy": 0.6442057237970422, + "roc-auc": 0.6063598206681031, + "score": 0.6063598206681031 + } + }, + "time": "0:02:05.402786", + "diversity": { + "train": { + "mean_centered_corr": -0.010739956051111221, + "mean_pairwise_disagreement": 0.02668459002628946, + "member_std": 0.029851045459508896 + }, + "val": { + "mean_centered_corr": -0.013114765286445618, + "mean_pairwise_disagreement": 0.09411593809791634, + "member_std": 0.06746508926153183 + }, + "test": { + "mean_centered_corr": -0.01157404389232397, + "mean_pairwise_disagreement": 0.059169786185915216, + "member_std": 0.06565776467323303 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 22, + 24, + 24 + ], + "head_selection_time": "0:00:20.703567" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.05-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.05-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.05-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.05-best-head-evaluation/0/report.json new file mode 100644 index 000000000..72357a786 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.05-best-head-evaluation/0/report.json @@ -0,0 +1,390 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.784937732846757, + "recall": 0.978628874184773, + "f1-score": 0.8711468302225982, + "support": 84179.0 + }, + "1": { + "precision": 0.5902050113895216, + "recall": 0.10297273666640172, + "f1-score": 0.17535192203573363, + "support": 25162.0 + }, + "accuracy": 0.7771192873670444, + "macro avg": { + "precision": 0.6875713721181393, + "recall": 0.5408008054255873, + "f1-score": 0.5232493761291659, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.740125039188322, + "recall": 0.7771192873670444, + "f1-score": 0.7110276482158681, + "support": 109341.0 + }, + "cross-entropy": 0.4989568858959861, + "roc-auc": 0.6810795630080747, + "score": 0.6810795630080747 + }, + "val": { + "0": { + "precision": 0.6882539682539682, + "recall": 0.9081067147828329, + "f1-score": 0.783041146889673, + "support": 15518.0 + }, + "1": { + "precision": 0.6233491811938722, + "recall": 0.26993022989820425, + "f1-score": 0.3767259956899992, + "support": 8743.0 + }, + "accuracy": 0.6781253864226536, + "macro avg": { + "precision": 0.6558015747239202, + "recall": 0.5890184723405185, + "f1-score": 0.579883571289836, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6648640604485843, + "recall": 0.6781253864226536, + "f1-score": 0.6366162935473232, + "support": 24261.0 + }, + "cross-entropy": 0.6298773246042463, + "roc-auc": 0.6410202011331968, + "score": 0.6410202011331968 + }, + "test": { + "0": { + "precision": 0.6683168316831684, + "recall": 0.9413780876964898, + "f1-score": 0.781687030766966, + "support": 16922.0 + }, + "1": { + "precision": 0.62122947689958, + "recall": 0.17067030315745305, + "f1-score": 0.26777485187623434, + "support": 9533.0 + }, + "accuracy": 0.6636552636552636, + "macro avg": { + "precision": 0.6447731542913742, + "recall": 0.5560241954269715, + "f1-score": 0.5247309413216001, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6513490088462018, + "recall": 0.6636552636552636, + "f1-score": 0.5964999280882533, + "support": 26455.0 + }, + "cross-entropy": 0.6607089123106477, + "roc-auc": 0.6024701571918213, + "score": 0.6024701571918213 + } + }, + "time": "0:01:11.748716", + "diversity": { + "train": { + "mean_centered_corr": -0.014211982488632202, + "mean_pairwise_disagreement": 0.02565772703979353, + "member_std": 0.02677466720342636 + }, + "val": { + "mean_centered_corr": -0.005956840701401234, + "mean_pairwise_disagreement": 0.08571943089333604, + "member_std": 0.0601416639983654 + }, + "test": { + "mean_centered_corr": 0.00013841522741131485, + "mean_pairwise_disagreement": 0.05374807189323318, + "member_std": 0.05852190777659416 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0 + ], + "head_selection_time": "0:00:04.721148" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.05-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.05-evaluation/0.toml new file mode 100644 index 000000000..0e6ac6970 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.05-evaluation/0.toml @@ -0,0 +1,44 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 + +[cf_fisd] +lambda = 0.05 +variant = "raw" +dataset_name = "ecom-offers" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.05-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.05-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.05-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.05-evaluation/0/report.json new file mode 100644 index 000000000..0ed76a5a7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.05-evaluation/0/report.json @@ -0,0 +1,386 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7863951555903647, + "recall": 0.97808241960584, + "f1-score": 0.8718266385001879, + "support": 84179.0 + }, + "1": { + "precision": 0.602627611458109, + "recall": 0.11119942770844925, + "f1-score": 0.18775373259520214, + "support": 25162.0 + }, + "accuracy": 0.7785917450910455, + "macro avg": { + "precision": 0.6945113835242368, + "recall": 0.5446409236571446, + "f1-score": 0.5297901855476951, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7441058135735931, + "recall": 0.7785917450910455, + "f1-score": 0.7144049718025973, + "support": 109341.0 + }, + "cross-entropy": 0.49233822908120295, + "roc-auc": 0.6988844522375441, + "score": 0.6988844522375441 + }, + "val": { + "0": { + "precision": 0.6753571753571753, + "recall": 0.9565021265627014, + "f1-score": 0.7917111158523575, + "support": 15518.0 + }, + "1": { + "precision": 0.7043363994743759, + "recall": 0.1839185634221663, + "f1-score": 0.2916742245601306, + "support": 8743.0 + }, + "accuracy": 0.6780841680062653, + "macro avg": { + "precision": 0.6898467874157757, + "recall": 0.5702103449924338, + "f1-score": 0.5416926702062441, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6858004941180131, + "recall": 0.6780841680062653, + "f1-score": 0.6115115139988503, + "support": 24261.0 + }, + "cross-entropy": 0.6866508172905287, + "roc-auc": 0.6228186386127663, + "score": 0.6228186386127663 + }, + "test": { + "0": { + "precision": 0.6564383127180463, + "recall": 0.97848954024347, + "f1-score": 0.7857447919138234, + "support": 16922.0 + }, + "1": { + "precision": 0.7043054427294883, + "recall": 0.09094723591733976, + "f1-score": 0.16109253065774806, + "support": 9533.0 + }, + "accuracy": 0.6586656586656586, + "macro avg": { + "precision": 0.6803718777237673, + "recall": 0.5347183880804048, + "f1-score": 0.47341866128578575, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6736871258119445, + "recall": 0.6586656586656586, + "f1-score": 0.5606527485740326, + "support": 26455.0 + }, + "cross-entropy": 0.7554621625639649, + "roc-auc": 0.5911736900637132, + "score": 0.5911736900637132 + } + }, + "time": "0:01:11.748716", + "diversity": { + "train": { + "mean_centered_corr": -0.014211982488632202, + "mean_pairwise_disagreement": 0.02565772703979353, + "member_std": 0.02677466720342636 + }, + "val": { + "mean_centered_corr": -0.005956840701401234, + "mean_pairwise_disagreement": 0.08571943089333604, + "member_std": 0.0601416639983654 + }, + "test": { + "mean_centered_corr": 0.00013841522741131485, + "mean_pairwise_disagreement": 0.05374807189323318, + "member_std": 0.05852190777659416 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.05-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.05-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.05-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.05-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..edf46fa0f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.05-greedy-heads-evaluation/0/report.json @@ -0,0 +1,391 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7827118483860296, + "recall": 0.9837013982109553, + "f1-score": 0.8717719266847043, + "support": 84179.0 + }, + "1": { + "precision": 0.6130851663846588, + "recall": 0.08640012717590016, + "f1-score": 0.15145604012818725, + "support": 25162.0 + }, + "accuracy": 0.7772107443685351, + "macro avg": { + "precision": 0.6978985073853442, + "recall": 0.5350507626934277, + "f1-score": 0.5116139834064457, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7436766596414737, + "recall": 0.7772107443685351, + "f1-score": 0.7060098764241883, + "support": 109341.0 + }, + "cross-entropy": 0.4979833881789, + "roc-auc": 0.6847415093580901, + "score": 0.6847415093580901 + }, + "val": { + "0": { + "precision": 0.6773795373502789, + "recall": 0.9548266529191907, + "f1-score": 0.7925224646983312, + "support": 15518.0 + }, + "1": { + "precision": 0.7063259321323837, + "recall": 0.19283998627473406, + "f1-score": 0.30296495956873315, + "support": 8743.0 + }, + "accuracy": 0.6802275256584642, + "macro avg": { + "precision": 0.6918527347413312, + "recall": 0.5738333195969624, + "f1-score": 0.5477437121335322, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6878110253177965, + "recall": 0.6802275256584642, + "f1-score": 0.6160993466344395, + "support": 24261.0 + }, + "cross-entropy": 0.6392793150076335, + "roc-auc": 0.6419149054445072, + "score": 0.6419149054445072 + }, + "test": { + "0": { + "precision": 0.658847933620552, + "recall": 0.9760075641177166, + "f1-score": 0.7866634913074542, + "support": 16922.0 + }, + "1": { + "precision": 0.7072819033886085, + "recall": 0.10290569600335676, + "f1-score": 0.17967032967032967, + "support": 9533.0 + }, + "accuracy": 0.6613872613872613, + "macro avg": { + "precision": 0.6830649185045803, + "recall": 0.5394566300605367, + "f1-score": 0.48316691048889193, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6763010061512224, + "recall": 0.6613872613872613, + "f1-score": 0.5679348649651103, + "support": 26455.0 + }, + "cross-entropy": 0.6803501878150763, + "roc-auc": 0.6041224275423289, + "score": 0.6041224275423289 + } + }, + "time": "0:01:11.748716", + "diversity": { + "train": { + "mean_centered_corr": -0.014211982488632202, + "mean_pairwise_disagreement": 0.02565772703979353, + "member_std": 0.02677466720342636 + }, + "val": { + "mean_centered_corr": -0.005956840701401234, + "mean_pairwise_disagreement": 0.08571943089333604, + "member_std": 0.0601416639983654 + }, + "test": { + "mean_centered_corr": 0.00013841522741131485, + "mean_pairwise_disagreement": 0.05374807189323318, + "member_std": 0.05852190777659416 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0, + 25 + ], + "head_selection_time": "0:00:15.652537" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.1-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.1-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.1-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.1-best-head-evaluation/0/report.json new file mode 100644 index 000000000..e14858863 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.1-best-head-evaluation/0/report.json @@ -0,0 +1,390 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7849017777184993, + "recall": 0.9787239097637178, + "f1-score": 0.8711623357741415, + "support": 84179.0 + }, + "1": { + "precision": 0.5906285714285714, + "recall": 0.10269453938478658, + "f1-score": 0.17496699055422013, + "support": 25162.0 + }, + "accuracy": 0.7771284330671935, + "macro avg": { + "precision": 0.6877651745735354, + "recall": 0.5407092245742522, + "f1-score": 0.5230646631641808, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7401948295776631, + "recall": 0.7771284330671935, + "f1-score": 0.7109510035527089, + "support": 109341.0 + }, + "cross-entropy": 0.49898712269253065, + "roc-auc": 0.681041160411764, + "score": 0.681041160411764 + }, + "val": { + "0": { + "precision": 0.6882634613505326, + "recall": 0.9077200670189457, + "f1-score": 0.7829035126722989, + "support": 15518.0 + }, + "1": { + "precision": 0.6226613965744401, + "recall": 0.27027336154638, + "f1-score": 0.3769341202743659, + "support": 8743.0 + }, + "accuracy": 0.6780017311734883, + "macro avg": { + "precision": 0.6554624289624864, + "recall": 0.5889967142826629, + "f1-score": 0.5799188164733324, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6646222737516135, + "recall": 0.6780017311734883, + "f1-score": 0.6366032613333134, + "support": 24261.0 + }, + "cross-entropy": 0.630002257594146, + "roc-auc": 0.640920668337369, + "score": 0.640920668337369 + }, + "test": { + "0": { + "precision": 0.668389645318541, + "recall": 0.9399007209549699, + "f1-score": 0.7812269757846653, + "support": 16922.0 + }, + "1": { + "precision": 0.6175253854832644, + "recall": 0.17224378474771845, + "f1-score": 0.2693569553805774, + "support": 9533.0 + }, + "accuracy": 0.6632772632772633, + "macro avg": { + "precision": 0.6429575154009026, + "recall": 0.5560722528513442, + "f1-score": 0.5252919655826214, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6500608232051526, + "recall": 0.6632772632772633, + "f1-score": 0.5967757603428898, + "support": 26455.0 + }, + "cross-entropy": 0.6609475697303346, + "roc-auc": 0.6022998594088651, + "score": 0.6022998594088651 + } + }, + "time": "0:01:12.374421", + "diversity": { + "train": { + "mean_centered_corr": -0.01448837574571371, + "mean_pairwise_disagreement": 0.025696983482570507, + "member_std": 0.02678096853196621 + }, + "val": { + "mean_centered_corr": -0.006191454362124205, + "mean_pairwise_disagreement": 0.0848030690435067, + "member_std": 0.060151606798172 + }, + "test": { + "mean_centered_corr": 2.267197305627633e-05, + "mean_pairwise_disagreement": 0.053483014370111145, + "member_std": 0.05869301036000252 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0 + ], + "head_selection_time": "0:00:04.736645" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.1-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.1-evaluation/0.toml new file mode 100644 index 000000000..6d8c4a5f0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.1-evaluation/0.toml @@ -0,0 +1,44 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "ecom-offers" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.1-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.1-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.1-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.1-evaluation/0/report.json new file mode 100644 index 000000000..d31884ffd --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.1-evaluation/0/report.json @@ -0,0 +1,386 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7863576019789309, + "recall": 0.97808241960584, + "f1-score": 0.8718035598945373, + "support": 84179.0 + }, + "1": { + "precision": 0.6021992238033635, + "recall": 0.11100071536443844, + "f1-score": 0.1874496644295302, + "support": 25162.0 + }, + "accuracy": 0.7785460165903001, + "macro avg": { + "precision": 0.6942784128911472, + "recall": 0.5445415674851392, + "f1-score": 0.5296266121620338, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.743978319626898, + "recall": 0.7785460165903001, + "f1-score": 0.7143172307253279, + "support": 109341.0 + }, + "cross-entropy": 0.4923443699583362, + "roc-auc": 0.6988661347453451, + "score": 0.6988661347453451 + }, + "val": { + "0": { + "precision": 0.6754186385147434, + "recall": 0.9565021265627014, + "f1-score": 0.791753347202219, + "support": 15518.0 + }, + "1": { + "precision": 0.7045951859956237, + "recall": 0.18414731785428343, + "f1-score": 0.2919840406238665, + "support": 8743.0 + }, + "accuracy": 0.6781666048390421, + "macro avg": { + "precision": 0.6900069122551835, + "recall": 0.5703247222084924, + "f1-score": 0.5418686939130428, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6859330672120492, + "recall": 0.6781666048390421, + "f1-score": 0.6116501755516467, + "support": 24261.0 + }, + "cross-entropy": 0.6863324565732282, + "roc-auc": 0.6228570653182646, + "score": 0.6228570653182646 + }, + "test": { + "0": { + "precision": 0.6563998890078091, + "recall": 0.9785486349131308, + "f1-score": 0.7857363164013381, + "support": 16922.0 + }, + "1": { + "precision": 0.7043973941368078, + "recall": 0.09073743837197105, + "f1-score": 0.16076572809218473, + "support": 9533.0 + }, + "accuracy": 0.6586278586278587, + "macro avg": { + "precision": 0.6803986415723084, + "recall": 0.534643036642551, + "f1-score": 0.4732510222467614, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6736956824833239, + "recall": 0.6586278586278587, + "f1-score": 0.5605295646209125, + "support": 26455.0 + }, + "cross-entropy": 0.7549517436640245, + "roc-auc": 0.5912083608375949, + "score": 0.5912083608375949 + } + }, + "time": "0:01:12.374421", + "diversity": { + "train": { + "mean_centered_corr": -0.01448837574571371, + "mean_pairwise_disagreement": 0.025696983482570507, + "member_std": 0.02678096853196621 + }, + "val": { + "mean_centered_corr": -0.006191454362124205, + "mean_pairwise_disagreement": 0.0848030690435067, + "member_std": 0.060151606798172 + }, + "test": { + "mean_centered_corr": 2.267197305627633e-05, + "mean_pairwise_disagreement": 0.053483014370111145, + "member_std": 0.05869301036000252 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.1-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.1-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.1-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.1-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..88e592e50 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.1-greedy-heads-evaluation/0/report.json @@ -0,0 +1,391 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7827204839776917, + "recall": 0.9836657598688509, + "f1-score": 0.871763287694308, + "support": 84179.0 + }, + "1": { + "precision": 0.6127851309490284, + "recall": 0.0864796121135045, + "f1-score": 0.15156897572528122, + "support": 25162.0 + }, + "accuracy": 0.777201598668386, + "macro avg": { + "precision": 0.6977528074633601, + "recall": 0.5350726859911776, + "f1-score": 0.5116661317097946, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7436142625885768, + "recall": 0.777201598668386, + "f1-score": 0.7060292146771904, + "support": 109341.0 + }, + "cross-entropy": 0.49797948962611205, + "roc-auc": 0.6847429264219673, + "score": 0.6847429264219673 + }, + "val": { + "0": { + "precision": 0.6772866480778901, + "recall": 0.9548266529191907, + "f1-score": 0.7924588848776575, + "support": 15518.0 + }, + "1": { + "precision": 0.7059563758389261, + "recall": 0.19249685462655838, + "f1-score": 0.3025074143974117, + "support": 8743.0 + }, + "accuracy": 0.6801038704092989, + "macro avg": { + "precision": 0.6916215119584082, + "recall": 0.5736617537728745, + "f1-score": 0.5474831496375345, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6876184328276835, + "recall": 0.6801038704092989, + "f1-score": 0.6158937924903367, + "support": 24261.0 + }, + "cross-entropy": 0.6393924934998982, + "roc-auc": 0.6418041361448852, + "score": 0.6418041361448852 + }, + "test": { + "0": { + "precision": 0.6587311081867847, + "recall": 0.9761848481266989, + "f1-score": 0.7866377770899308, + "support": 16922.0 + }, + "1": { + "precision": 0.7075471698113207, + "recall": 0.1022763033672506, + "f1-score": 0.17871872422326093, + "support": 9533.0 + }, + "accuracy": 0.6612738612738612, + "macro avg": { + "precision": 0.6831391389990527, + "recall": 0.5392305757469747, + "f1-score": 0.48267825065659586, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6763218666621845, + "recall": 0.6612738612738612, + "f1-score": 0.5675755079166946, + "support": 26455.0 + }, + "cross-entropy": 0.6805522467825859, + "roc-auc": 0.6039617691395597, + "score": 0.6039617691395597 + } + }, + "time": "0:01:12.374421", + "diversity": { + "train": { + "mean_centered_corr": -0.01448837574571371, + "mean_pairwise_disagreement": 0.025696983482570507, + "member_std": 0.02678096853196621 + }, + "val": { + "mean_centered_corr": -0.006191454362124205, + "mean_pairwise_disagreement": 0.0848030690435067, + "member_std": 0.060151606798172 + }, + "test": { + "mean_centered_corr": 2.267197305627633e-05, + "mean_pairwise_disagreement": 0.053483014370111145, + "member_std": 0.05869301036000252 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0, + 25 + ], + "head_selection_time": "0:00:15.620571" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.2-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.2-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.2-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.2-best-head-evaluation/0/report.json new file mode 100644 index 000000000..5bc424be9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.2-best-head-evaluation/0/report.json @@ -0,0 +1,390 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.2, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.2, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7802069833399072, + "recall": 0.9869207284477126, + "f1-score": 0.8714734528823409, + "support": 84179.0 + }, + "1": { + "precision": 0.614900314795383, + "recall": 0.06986726015420078, + "f1-score": 0.12547732058099284, + "support": 25162.0 + }, + "accuracy": 0.7758846178469193, + "macro avg": { + "precision": 0.697553649067645, + "recall": 0.5283939943009567, + "f1-score": 0.49847538673166686, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7421659338349884, + "recall": 0.7758846178469193, + "f1-score": 0.6998017589983769, + "support": 109341.0 + }, + "cross-entropy": 0.49962618774719303, + "roc-auc": 0.6840523798402088, + "score": 0.6840523798402088 + }, + "val": { + "0": { + "precision": 0.6743785937429257, + "recall": 0.9598530738497228, + "f1-score": 0.7921818907060232, + "support": 15518.0 + }, + "1": { + "precision": 0.7134314627414904, + "recall": 0.17739906210682832, + "f1-score": 0.2841439956031877, + "support": 8743.0 + }, + "accuracy": 0.677878075924323, + "macro avg": { + "precision": 0.6939050282422081, + "recall": 0.5686260679782755, + "f1-score": 0.5381629431546054, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6884521782470455, + "recall": 0.677878075924323, + "f1-score": 0.6090989461907892, + "support": 24261.0 + }, + "cross-entropy": 0.6483381636613907, + "roc-auc": 0.638077626500147, + "score": 0.638077626500147 + }, + "test": { + "0": { + "precision": 0.6575799721835883, + "recall": 0.977898593546862, + "f1-score": 0.7863710884596193, + "support": 16922.0 + }, + "1": { + "precision": 0.710077519379845, + "recall": 0.09608727577887338, + "f1-score": 0.16926914903446363, + "support": 9533.0 + }, + "accuracy": 0.6601398601398601, + "macro avg": { + "precision": 0.6838287457817167, + "recall": 0.5369929346628677, + "f1-score": 0.4778201187470415, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6764973457395103, + "recall": 0.6601398601398601, + "f1-score": 0.5639997866815052, + "support": 26455.0 + }, + "cross-entropy": 0.6867257666439333, + "roc-auc": 0.6005513130366958, + "score": 0.6005513130366958 + } + }, + "time": "0:01:07.664836", + "diversity": { + "train": { + "mean_centered_corr": -0.017361776903271675, + "mean_pairwise_disagreement": 0.024792702380330725, + "member_std": 0.027867170050740242 + }, + "val": { + "mean_centered_corr": -0.004654714372009039, + "mean_pairwise_disagreement": 0.06891403433892973, + "member_std": 0.05895354598760605 + }, + "test": { + "mean_centered_corr": 0.0006982876220718026, + "mean_pairwise_disagreement": 0.0455133031745935, + "member_std": 0.06036552041769028 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 11 + ], + "head_selection_time": "0:00:04.884249" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.2-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.2-evaluation/0.toml new file mode 100644 index 000000000..4a734c33a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.2-evaluation/0.toml @@ -0,0 +1,44 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 + +[cf_fisd] +lambda = 0.2 +variant = "raw" +dataset_name = "ecom-offers" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.2-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.2-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.2-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.2-evaluation/0/report.json new file mode 100644 index 000000000..5b73160af --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.2-evaluation/0/report.json @@ -0,0 +1,386 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.2, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.2, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.781663133097762, + "recall": 0.9854476769740672, + "f1-score": 0.8718051118210862, + "support": 84179.0 + }, + "1": { + "precision": 0.619092039800995, + "recall": 0.07912725538510453, + "f1-score": 0.14031996617097753, + "support": 25162.0 + }, + "accuracy": 0.7768814991631684, + "macro avg": { + "precision": 0.7003775864493785, + "recall": 0.5322874661795859, + "f1-score": 0.5060625389960318, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7442516054042778, + "recall": 0.7768814991631684, + "f1-score": 0.703472745784119, + "support": 109341.0 + }, + "cross-entropy": 0.49560710251264434, + "roc-auc": 0.6917690308083511, + "score": 0.6917690308083511 + }, + "val": { + "0": { + "precision": 0.6751520101642617, + "recall": 0.958822013146024, + "f1-score": 0.7923634039833848, + "support": 15518.0 + }, + "1": { + "precision": 0.7125506072874493, + "recall": 0.18117351023676084, + "f1-score": 0.28889294182017144, + "support": 8743.0 + }, + "accuracy": 0.6785787890029265, + "macro avg": { + "precision": 0.6938513087258555, + "recall": 0.5699977616913924, + "f1-score": 0.5406281729017781, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6886294403875842, + "recall": 0.6785787890029265, + "f1-score": 0.6109264372180835, + "support": 24261.0 + }, + "cross-entropy": 0.6738878510736607, + "roc-auc": 0.622913421046708, + "score": 0.622913421046708 + }, + "test": { + "0": { + "precision": 0.6571337908056007, + "recall": 0.9790213922704172, + "f1-score": 0.7864144494814042, + "support": 16922.0 + }, + "1": { + "precision": 0.7146302250803859, + "recall": 0.09325500891639568, + "f1-score": 0.16498097800872227, + "support": 9533.0 + }, + "accuracy": 0.6598374598374598, + "macro avg": { + "precision": 0.6858820079429933, + "recall": 0.5361382005934064, + "f1-score": 0.4756977137450632, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6778525021245018, + "recall": 0.6598374598374598, + "f1-score": 0.5624822898310894, + "support": 26455.0 + }, + "cross-entropy": 0.7343241435689584, + "roc-auc": 0.5913362546461658, + "score": 0.5913362546461658 + } + }, + "time": "0:01:07.664836", + "diversity": { + "train": { + "mean_centered_corr": -0.017361776903271675, + "mean_pairwise_disagreement": 0.024792702380330725, + "member_std": 0.027867170050740242 + }, + "val": { + "mean_centered_corr": -0.004654714372009039, + "mean_pairwise_disagreement": 0.06891403433892973, + "member_std": 0.05895354598760605 + }, + "test": { + "mean_centered_corr": 0.0006982876220718026, + "mean_pairwise_disagreement": 0.0455133031745935, + "member_std": 0.06036552041769028 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.2-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.2-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.2-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.2-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..56666b5b1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.2-greedy-heads-evaluation/0/report.json @@ -0,0 +1,392 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.2, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.2, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7804729171362031, + "recall": 0.9873127502108602, + "f1-score": 0.8717921821814997, + "support": 84179.0 + }, + "1": { + "precision": 0.6256572029442692, + "recall": 0.07094030681185916, + "f1-score": 0.1274317330001785, + "support": 25162.0 + }, + "accuracy": 0.7764333598558638, + "macro avg": { + "precision": 0.7030650600402362, + "recall": 0.5291265285113597, + "f1-score": 0.4996119575908391, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7448460891348364, + "recall": 0.7764333598558638, + "f1-score": 0.7004968984151138, + "support": 109341.0 + }, + "cross-entropy": 0.4989856299511063, + "roc-auc": 0.6835209664866834, + "score": 0.6835209664866834 + }, + "val": { + "0": { + "precision": 0.678697546434304, + "recall": 0.9536667096275293, + "f1-score": 0.7930230688851378, + "support": 15518.0 + }, + "1": { + "precision": 0.7072475570032574, + "recall": 0.1986732242937207, + "f1-score": 0.3102062684168229, + "support": 8743.0 + }, + "accuracy": 0.6815877333992828, + "macro avg": { + "precision": 0.6929725517187807, + "recall": 0.5761699669606251, + "f1-score": 0.5516146686509804, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6889861883865879, + "recall": 0.6815877333992828, + "f1-score": 0.6190291161834983, + "support": 24261.0 + }, + "cross-entropy": 0.6383314262513924, + "roc-auc": 0.6387008784019832, + "score": 0.6387008784019832 + }, + "test": { + "0": { + "precision": 0.661681942434475, + "recall": 0.972698262616712, + "f1-score": 0.7875974927030002, + "support": 16922.0 + }, + "1": { + "precision": 0.707409753008233, + "recall": 0.11717192908842966, + "f1-score": 0.20104391648668107, + "support": 9533.0 + }, + "accuracy": 0.6644112644112644, + "macro avg": { + "precision": 0.684545847721354, + "recall": 0.5449350958525708, + "f1-score": 0.4943207045948406, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6781598565603354, + "recall": 0.6644112644112644, + "f1-score": 0.5762342251894802, + "support": 26455.0 + }, + "cross-entropy": 0.676206711376993, + "roc-auc": 0.6023730040175572, + "score": 0.6023730040175572 + } + }, + "time": "0:01:07.664836", + "diversity": { + "train": { + "mean_centered_corr": -0.017361776903271675, + "mean_pairwise_disagreement": 0.024792702380330725, + "member_std": 0.027867170050740242 + }, + "val": { + "mean_centered_corr": -0.004654714372009039, + "mean_pairwise_disagreement": 0.06891403433892973, + "member_std": 0.05895354598760605 + }, + "test": { + "mean_centered_corr": 0.0006982876220718026, + "mean_pairwise_disagreement": 0.0455133031745935, + "member_std": 0.06036552041769028 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 11, + 17, + 0 + ], + "head_selection_time": "0:00:20.704412" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_cf_fisd-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_cf_fisd-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_cf_fisd-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_cf_fisd-best-head-evaluation/0/report.json new file mode 100644 index 000000000..a4dec2eba --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_cf_fisd-best-head-evaluation/0/report.json @@ -0,0 +1,400 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_cf_fisd-evaluation/_esam_diagnostics/esam_cf_fisd-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7849948064076541, + "recall": 0.9785694769479324, + "f1-score": 0.8711584424374458, + "support": 84179.0 + }, + "1": { + "precision": 0.5903723887375113, + "recall": 0.10333041888562118, + "f1-score": 0.17587769735507, + "support": 25162.0 + }, + "accuracy": 0.7771558701676406, + "macro avg": { + "precision": 0.6876835975725827, + "recall": 0.5409499479167768, + "f1-score": 0.5235180698962579, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.740207496309739, + "recall": 0.7771558701676406, + "f1-score": 0.7111575817560661, + "support": 109341.0 + }, + "cross-entropy": 0.49905380562363294, + "roc-auc": 0.6808058298435642, + "score": 0.6808058298435642 + }, + "val": { + "0": { + "precision": 0.6883192826860289, + "recall": 0.9102332774842119, + "f1-score": 0.783873026443575, + "support": 15518.0 + }, + "1": { + "precision": 0.6275401069518717, + "recall": 0.268443326089443, + "f1-score": 0.3760314027076824, + "support": 8743.0 + }, + "accuracy": 0.6789497547504225, + "macro avg": { + "precision": 0.6579296948189504, + "recall": 0.5893383017868274, + "f1-score": 0.5799522145756287, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.666416132220519, + "recall": 0.6789497547504225, + "f1-score": 0.6368980742024098, + "support": 24261.0 + }, + "cross-entropy": 0.6300629395883726, + "roc-auc": 0.6410485079831951, + "score": 0.6410485079831951 + }, + "test": { + "0": { + "precision": 0.6681612470667113, + "recall": 0.9422645077414017, + "f1-score": 0.781885941254352, + "support": 16922.0 + }, + "1": { + "precision": 0.6229255113855654, + "recall": 0.16930661911255637, + "f1-score": 0.26624876278455956, + "support": 9533.0 + }, + "accuracy": 0.6637308637308638, + "macro avg": { + "precision": 0.6455433792261384, + "recall": 0.555785563426979, + "f1-score": 0.5240673520194558, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6518606510263272, + "recall": 0.6637308637308638, + "f1-score": 0.5960772388407238, + "support": 26455.0 + }, + "cross-entropy": 0.6611287604378554, + "roc-auc": 0.6024310820580536, + "score": 0.6024310820580536 + } + }, + "time": "0:01:57.344730", + "diversity": { + "train": { + "mean_centered_corr": -0.014402652159333229, + "mean_pairwise_disagreement": 0.025755084492993362, + "member_std": 0.02682984620332718 + }, + "val": { + "mean_centered_corr": -0.0061649116687476635, + "mean_pairwise_disagreement": 0.08504215247888885, + "member_std": 0.060025766491889954 + }, + "test": { + "mean_centered_corr": -0.0001455514138797298, + "mean_pairwise_disagreement": 0.05350869705708415, + "member_std": 0.05860954150557518 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0 + ], + "head_selection_time": "0:00:04.814924" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_cf_fisd-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_cf_fisd-evaluation/0.toml new file mode 100644 index 000000000..025615a45 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_cf_fisd-evaluation/0.toml @@ -0,0 +1,54 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "ecom-offers" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_cf_fisd-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_cf_fisd-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_cf_fisd-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_cf_fisd-evaluation/0/report.json new file mode 100644 index 000000000..97857fc20 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_cf_fisd-evaluation/0/report.json @@ -0,0 +1,396 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_cf_fisd-evaluation/_esam_diagnostics/esam_cf_fisd-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7863774428334002, + "recall": 0.9780230223689994, + "f1-score": 0.8717921566343522, + "support": 84179.0 + }, + "1": { + "precision": 0.6018936948568969, + "recall": 0.11115968523964709, + "f1-score": 0.18766144453017544, + "support": 25162.0 + }, + "accuracy": 0.778536870890151, + "macro avg": { + "precision": 0.6941355688451485, + "recall": 0.5445913538043232, + "f1-score": 0.5297268005822638, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.743923285046433, + "recall": 0.778536870890151, + "f1-score": 0.7143571873367851, + "support": 109341.0 + }, + "cross-entropy": 0.49238150256808705, + "roc-auc": 0.6987779918614105, + "score": 0.6987779918614105 + }, + "val": { + "0": { + "precision": 0.6755440225803514, + "recall": 0.9562443613867766, + "f1-score": 0.7917511471561199, + "support": 15518.0 + }, + "1": { + "precision": 0.7041394335511982, + "recall": 0.1848335811506348, + "f1-score": 0.29280666787461496, + "support": 8743.0 + }, + "accuracy": 0.678249041671819, + "macro avg": { + "precision": 0.6898417280657748, + "recall": 0.5705389712687057, + "f1-score": 0.5422789075153674, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6858490255941642, + "recall": 0.678249041671819, + "f1-score": 0.6119452206750104, + "support": 24261.0 + }, + "cross-entropy": 0.686338046017919, + "roc-auc": 0.6227702320934685, + "score": 0.6227702320934685 + }, + "test": { + "0": { + "precision": 0.6565536387071188, + "recall": 0.9783122562344877, + "f1-score": 0.7857702256923844, + "support": 16922.0 + }, + "1": { + "precision": 0.7040322580645161, + "recall": 0.09157662855344592, + "f1-score": 0.1620718462823726, + "support": 9533.0 + }, + "accuracy": 0.6587790587790587, + "macro avg": { + "precision": 0.6802929483858174, + "recall": 0.5349444423939668, + "f1-score": 0.4739210359873785, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6736624528569607, + "recall": 0.6587790587790587, + "f1-score": 0.5610219115394589, + "support": 26455.0 + }, + "cross-entropy": 0.754977849360994, + "roc-auc": 0.5911624823470715, + "score": 0.5911624823470715 + } + }, + "time": "0:01:57.344730", + "diversity": { + "train": { + "mean_centered_corr": -0.014402652159333229, + "mean_pairwise_disagreement": 0.025755084492993362, + "member_std": 0.02682984620332718 + }, + "val": { + "mean_centered_corr": -0.0061649116687476635, + "mean_pairwise_disagreement": 0.08504215247888885, + "member_std": 0.060025766491889954 + }, + "test": { + "mean_centered_corr": -0.0001455514138797298, + "mean_pairwise_disagreement": 0.05350869705708415, + "member_std": 0.05860954150557518 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_cf_fisd-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_cf_fisd-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_cf_fisd-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_cf_fisd-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..a5b8aaea0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_cf_fisd-greedy-heads-evaluation/0/report.json @@ -0,0 +1,401 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_cf_fisd-evaluation/_esam_diagnostics/esam_cf_fisd-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7827360978514647, + "recall": 0.9837132776583233, + "f1-score": 0.871791632451098, + "support": 84179.0 + }, + "1": { + "precision": 0.6135851183765502, + "recall": 0.08651935458230665, + "f1-score": 0.15165447579240682, + "support": 25162.0 + }, + "accuracy": 0.7772473271691315, + "macro avg": { + "precision": 0.6981606081140075, + "recall": 0.535116316120315, + "f1-score": 0.5117230541217525, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.743810379726079, + "recall": 0.7772473271691315, + "f1-score": 0.7060707122487404, + "support": 109341.0 + }, + "cross-entropy": 0.49803582395239526, + "roc-auc": 0.6845751097530018, + "score": 0.6845751097530018 + }, + "val": { + "0": { + "precision": 0.6770657287717535, + "recall": 0.9552133006830777, + "f1-score": 0.7924407259896822, + "support": 15518.0 + }, + "1": { + "precision": 0.7065033783783784, + "recall": 0.19135308246597277, + "f1-score": 0.3011430114301143, + "support": 8743.0 + }, + "accuracy": 0.6799389967437451, + "macro avg": { + "precision": 0.691784553575066, + "recall": 0.5732831915745252, + "f1-score": 0.5467918687098983, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6876742515247612, + "recall": 0.6799389967437451, + "f1-score": 0.6153904841037623, + "support": 24261.0 + }, + "cross-entropy": 0.6393443379299147, + "roc-auc": 0.6417659932080955, + "score": 0.6417659932080955 + }, + "test": { + "0": { + "precision": 0.6586659224113871, + "recall": 0.9762439427963597, + "f1-score": 0.786610480203795, + "support": 16922.0 + }, + "1": { + "precision": 0.7074235807860262, + "recall": 0.10196160704919753, + "f1-score": 0.17823416154762997, + "support": 9533.0 + }, + "accuracy": 0.6611982611982612, + "macro avg": { + "precision": 0.6830447515987066, + "recall": 0.5391027749227786, + "f1-score": 0.48242232087571246, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6762356354064896, + "recall": 0.6611982611982612, + "f1-score": 0.5673834363274305, + "support": 26455.0 + }, + "cross-entropy": 0.6804458567708631, + "roc-auc": 0.6040036864957168, + "score": 0.6040036864957168 + } + }, + "time": "0:01:57.344730", + "diversity": { + "train": { + "mean_centered_corr": -0.014402652159333229, + "mean_pairwise_disagreement": 0.025755084492993362, + "member_std": 0.02682984620332718 + }, + "val": { + "mean_centered_corr": -0.0061649116687476635, + "mean_pairwise_disagreement": 0.08504215247888885, + "member_std": 0.060025766491889954 + }, + "test": { + "mean_centered_corr": -0.0001455514138797298, + "mean_pairwise_disagreement": 0.05350869705708415, + "member_std": 0.05860954150557518 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0, + 25 + ], + "head_selection_time": "0:00:15.908281" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_mfb-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_mfb-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_mfb-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_mfb-best-head-evaluation/0/report.json new file mode 100644 index 000000000..4541709e6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_mfb-best-head-evaluation/0/report.json @@ -0,0 +1,265 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.887499988079071, + "min_features_per_member": 92.0, + "mean_features_per_member": 97.625, + "max_features_per_member": 103.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.399999618530273, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.797659951359852, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_mfb-evaluation/_esam_diagnostics/esam_mfb-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7919478842655086, + "recall": 0.9618075767115314, + "f1-score": 0.8686518644085981, + "support": 84179.0 + }, + "1": { + "precision": 0.5476290980723231, + "recall": 0.15467768857801448, + "f1-score": 0.2412222256654994, + "support": 25162.0 + }, + "accuracy": 0.7760675318499007, + "macro avg": { + "precision": 0.6697884911689158, + "recall": 0.5582426326447729, + "f1-score": 0.5549370450370488, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7357242417325802, + "recall": 0.7760675318499007, + "f1-score": 0.7242651789927537, + "support": 109341.0 + }, + "cross-entropy": 0.501967718593187, + "roc-auc": 0.6776997459791548, + "score": 0.6776997459791548 + }, + "val": { + "0": { + "precision": 0.7099598084016957, + "recall": 0.8309704858873567, + "f1-score": 0.7657136070781746, + "support": 15518.0 + }, + "1": { + "precision": 0.5698589701541489, + "recall": 0.39746082580349995, + "f1-score": 0.46829728454955866, + "support": 8743.0 + }, + "accuracy": 0.6747454762788013, + "macro avg": { + "precision": 0.6399093892779223, + "recall": 0.6142156558454284, + "f1-score": 0.6170054458138666, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6594713030310061, + "recall": 0.6747454762788013, + "f1-score": 0.6585329093382757, + "support": 24261.0 + }, + "cross-entropy": 0.6236491778893606, + "roc-auc": 0.6500013075472437, + "score": 0.6500013075472437 + }, + "test": { + "0": { + "precision": 0.6826944581115534, + "recall": 0.9019619430327385, + "f1-score": 0.7771582779602332, + "support": 16922.0 + }, + "1": { + "precision": 0.5951683748169839, + "recall": 0.25584810657715307, + "f1-score": 0.3578607585650356, + "support": 9533.0 + }, + "accuracy": 0.6691362691362691, + "macro avg": { + "precision": 0.6389314164642687, + "recall": 0.5789050248049458, + "f1-score": 0.5675095182626344, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6511546300243438, + "recall": 0.6691362691362691, + "f1-score": 0.6260653559268022, + "support": 26455.0 + }, + "cross-entropy": 0.6442269054009311, + "roc-auc": 0.6046018053871005, + "score": 0.6046018053871005 + } + }, + "time": "0:01:42.652042", + "diversity": { + "train": { + "mean_centered_corr": -0.011058990843594074, + "mean_pairwise_disagreement": 0.026724399636414167, + "member_std": 0.03006140887737274 + }, + "val": { + "mean_centered_corr": -0.00816602073609829, + "mean_pairwise_disagreement": 0.09307434206764872, + "member_std": 0.06737758219242096 + }, + "test": { + "mean_centered_corr": -0.005533241201192141, + "mean_pairwise_disagreement": 0.05856963437608599, + "member_std": 0.06563688069581985 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 24 + ], + "head_selection_time": "0:00:04.775982" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_mfb-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_mfb-evaluation/0.toml new file mode 100644 index 000000000..aaf475ea7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_mfb-evaluation/0.toml @@ -0,0 +1,56 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_mfb-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_mfb-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_mfb-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_mfb-evaluation/0/report.json new file mode 100644 index 000000000..5cdd60541 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_mfb-evaluation/0/report.json @@ -0,0 +1,261 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.887499988079071, + "min_features_per_member": 92.0, + "mean_features_per_member": 97.625, + "max_features_per_member": 103.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.399999618530273, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.797659951359852, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_mfb-evaluation/_esam_diagnostics/esam_mfb-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7842988383569964, + "recall": 0.9817175305004812, + "f1-score": 0.8719737057299772, + "support": 84179.0 + }, + "1": { + "precision": 0.6126352881953184, + "recall": 0.09673316906446229, + "f1-score": 0.16708426291402093, + "support": 25162.0 + }, + "accuracy": 0.7780612944823991, + "macro avg": { + "precision": 0.6984670632761574, + "recall": 0.5392253497824717, + "f1-score": 0.5195289843219991, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7447949171456655, + "recall": 0.7780612944823991, + "f1-score": 0.7097616520617732, + "support": 109341.0 + }, + "cross-entropy": 0.49277000777319097, + "roc-auc": 0.6981352050298899, + "score": 0.6981352050298899 + }, + "val": { + "0": { + "precision": 0.6762921399571188, + "recall": 0.95534218327104, + "f1-score": 0.7919549133256765, + "support": 15518.0 + }, + "1": { + "precision": 0.7038461538461539, + "recall": 0.18837927484845018, + "f1-score": 0.2972119462239466, + "support": 8743.0 + }, + "accuracy": 0.6789497547504225, + "macro avg": { + "precision": 0.6900691469016363, + "recall": 0.571860729059745, + "f1-score": 0.5445834297748116, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6862218519818429, + "recall": 0.6789497547504225, + "f1-score": 0.6136630967735796, + "support": 24261.0 + }, + "cross-entropy": 0.6746455512551084, + "roc-auc": 0.6249124647240485, + "score": 0.6249124647240485 + }, + "test": { + "0": { + "precision": 0.6572768087726966, + "recall": 0.9776031201985581, + "f1-score": 0.786058777410848, + "support": 16922.0 + }, + "1": { + "precision": 0.7052877138413686, + "recall": 0.09514318682471415, + "f1-score": 0.16766799149644143, + "support": 9533.0 + }, + "accuracy": 0.6596106596106596, + "macro avg": { + "precision": 0.6812822613070326, + "recall": 0.5363731535116362, + "f1-score": 0.4768633844536447, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6745774308864615, + "recall": 0.6596106596106596, + "f1-score": 0.5632230426869002, + "support": 26455.0 + }, + "cross-entropy": 0.7368636466213944, + "roc-auc": 0.5926966501436739, + "score": 0.5926966501436739 + } + }, + "time": "0:01:42.652042", + "diversity": { + "train": { + "mean_centered_corr": -0.011058990843594074, + "mean_pairwise_disagreement": 0.026724399636414167, + "member_std": 0.03006140887737274 + }, + "val": { + "mean_centered_corr": -0.00816602073609829, + "mean_pairwise_disagreement": 0.09307434206764872, + "member_std": 0.06737758219242096 + }, + "test": { + "mean_centered_corr": -0.005533241201192141, + "mean_pairwise_disagreement": 0.05856963437608599, + "member_std": 0.06563688069581985 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_mfb-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_mfb-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_mfb-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_mfb-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..7e5a636a1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_mfb-greedy-heads-evaluation/0/report.json @@ -0,0 +1,266 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.887499988079071, + "min_features_per_member": 92.0, + "mean_features_per_member": 97.625, + "max_features_per_member": 103.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.399999618530273, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.797659951359852, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_mfb-evaluation/_esam_diagnostics/esam_mfb-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7903039369007956, + "recall": 0.9665237173166704, + "f1-score": 0.8695759052627079, + "support": 84179.0 + }, + "1": { + "precision": 0.559136420525657, + "recall": 0.14203958349892695, + "f1-score": 0.22653229384547124, + "support": 25162.0 + }, + "accuracy": 0.7767900421616777, + "macro avg": { + "precision": 0.6747201787132263, + "recall": 0.5542816504077986, + "f1-score": 0.5480540995540896, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7371067185926472, + "recall": 0.7767900421616777, + "f1-score": 0.7215960683261471, + "support": 109341.0 + }, + "cross-entropy": 0.49894859408324227, + "roc-auc": 0.6833477924522856, + "score": 0.6833477924522856 + }, + "val": { + "0": { + "precision": 0.7034465020576132, + "recall": 0.8812346951926795, + "f1-score": 0.7823674123233595, + "support": 15518.0 + }, + "1": { + "precision": 0.6177141671852313, + "recall": 0.3406153494223951, + "f1-score": 0.4391035092892952, + "support": 8743.0 + }, + "accuracy": 0.6864102881167305, + "macro avg": { + "precision": 0.6605803346214223, + "recall": 0.6109250223075373, + "f1-score": 0.6107354608063273, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6725509163938221, + "recall": 0.6864102881167305, + "f1-score": 0.6586645021289395, + "support": 24261.0 + }, + "cross-entropy": 0.6204343840179957, + "roc-auc": 0.650368091501537, + "score": 0.650368091501537 + }, + "test": { + "0": { + "precision": 0.676635434047833, + "recall": 0.9345822006854981, + "f1-score": 0.784960913264673, + "support": 16922.0 + }, + "1": { + "precision": 0.6408176508760545, + "recall": 0.2071750760516102, + "f1-score": 0.31311930241775665, + "support": 9533.0 + }, + "accuracy": 0.6724626724626724, + "macro avg": { + "precision": 0.6587265424619437, + "recall": 0.5708786383685541, + "f1-score": 0.5490401078412148, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6637285761012608, + "recall": 0.6724626724626724, + "f1-score": 0.6149338455571072, + "support": 26455.0 + }, + "cross-entropy": 0.6488325155298535, + "roc-auc": 0.6061843343570334, + "score": 0.6061843343570334 + } + }, + "time": "0:01:42.652042", + "diversity": { + "train": { + "mean_centered_corr": -0.011058990843594074, + "mean_pairwise_disagreement": 0.026724399636414167, + "member_std": 0.03006140887737274 + }, + "val": { + "mean_centered_corr": -0.00816602073609829, + "mean_pairwise_disagreement": 0.09307434206764872, + "member_std": 0.06737758219242096 + }, + "test": { + "mean_centered_corr": -0.005533241201192141, + "mean_pairwise_disagreement": 0.05856963437608599, + "member_std": 0.06563688069581985 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 24, + 22 + ], + "head_selection_time": "0:00:15.768684" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..223176d4b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.001-best-head-evaluation/0/report.json @@ -0,0 +1,229 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.001-evaluation/_esam_diagnostics/esam_rho0.001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7841790931223371, + "recall": 0.9795554710794854, + "f1-score": 0.8710459459887709, + "support": 84179.0 + }, + "1": { + "precision": 0.5891620911912151, + "recall": 0.0980844130037358, + "f1-score": 0.16817144219958435, + "support": 25162.0 + }, + "accuracy": 0.7767077308603361, + "macro avg": { + "precision": 0.6866705921567762, + "recall": 0.5388199420416105, + "f1-score": 0.5196086940941776, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7393009796736684, + "recall": 0.7767077308603361, + "f1-score": 0.709297578365066, + "support": 109341.0 + }, + "cross-entropy": 0.4992703210863206, + "roc-auc": 0.6803857932728635, + "score": 0.6803857932728635 + }, + "val": { + "0": { + "precision": 0.688133775351014, + "recall": 0.9095888645444, + "f1-score": 0.7835137385512073, + "support": 15518.0 + }, + "1": { + "precision": 0.6257668711656442, + "recall": 0.26832894887338443, + "f1-score": 0.3756003842459174, + "support": 8743.0 + }, + "accuracy": 0.6784963521701496, + "macro avg": { + "precision": 0.6569503232583291, + "recall": 0.5889589067088923, + "f1-score": 0.5795570613985623, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6656584510324497, + "recall": 0.6784963521701496, + "f1-score": 0.63651293657721, + "support": 24261.0 + }, + "cross-entropy": 0.6310932933009203, + "roc-auc": 0.6410241296714208, + "score": 0.6410241296714208 + }, + "test": { + "0": { + "precision": 0.6670702684647822, + "recall": 0.9441555371705472, + "f1-score": 0.781787488072811, + "support": 16922.0 + }, + "1": { + "precision": 0.6226038338658147, + "recall": 0.16353718661491662, + "f1-score": 0.2590346431835175, + "support": 9533.0 + }, + "accuracy": 0.6628614628614629, + "macro avg": { + "precision": 0.6448370511652985, + "recall": 0.553846361892732, + "f1-score": 0.5204110656281642, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6510468883463941, + "recall": 0.6628614628614629, + "f1-score": 0.5934146712015339, + "support": 26455.0 + }, + "cross-entropy": 0.6635058051130894, + "roc-auc": 0.6024966050474919, + "score": 0.6024966050474919 + } + }, + "time": "0:01:33.915789", + "diversity": { + "train": { + "mean_centered_corr": -0.013111491687595844, + "mean_pairwise_disagreement": 0.02585577938919114, + "member_std": 0.02714134193956852 + }, + "val": { + "mean_centered_corr": -0.004636597353965044, + "mean_pairwise_disagreement": 0.08690653790565238, + "member_std": 0.060862503945827484 + }, + "test": { + "mean_centered_corr": 0.0027443149592727423, + "mean_pairwise_disagreement": 0.05465580626870949, + "member_std": 0.05927826836705208 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0 + ], + "head_selection_time": "0:00:04.756597" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.001-evaluation/0.toml new file mode 100644 index 000000000..9b75584bc --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.001-evaluation/0.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.001 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.001-evaluation/0/report.json new file mode 100644 index 000000000..517e309dc --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.001-evaluation/0/report.json @@ -0,0 +1,225 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.001-evaluation/_esam_diagnostics/esam_rho0.001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7862810438170169, + "recall": 0.9782487318689934, + "f1-score": 0.8718225610078874, + "support": 84179.0 + }, + "1": { + "precision": 0.602819956616052, + "recall": 0.11044432080120817, + "f1-score": 0.18668547628644364, + "support": 25162.0 + }, + "accuracy": 0.7785460165903001, + "macro avg": { + "precision": 0.6945505002165344, + "recall": 0.5443465263351008, + "f1-score": 0.5292540186471655, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7440622249279388, + "recall": 0.7785460165903001, + "f1-score": 0.7141560011103103, + "support": 109341.0 + }, + "cross-entropy": 0.4922902286245573, + "roc-auc": 0.6989728271677539, + "score": 0.6989728271677539 + }, + "val": { + "0": { + "precision": 0.675189954047045, + "recall": 0.9563088026807578, + "f1-score": 0.791529989065792, + "support": 15518.0 + }, + "1": { + "precision": 0.7028921998247152, + "recall": 0.18346105455793205, + "f1-score": 0.2909750566893424, + "support": 8743.0 + }, + "accuracy": 0.677795639091546, + "macro avg": { + "precision": 0.6890410769358801, + "recall": 0.5698849286193449, + "f1-score": 0.5412525228775672, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6851730847850266, + "recall": 0.677795639091546, + "f1-score": 0.6111436993923531, + "support": 24261.0 + }, + "cross-entropy": 0.6872600691770208, + "roc-auc": 0.6227874903903754, + "score": 0.6227874903903754 + }, + "test": { + "0": { + "precision": 0.656414727914074, + "recall": 0.9787259189221132, + "f1-score": 0.7858040946077385, + "support": 16922.0 + }, + "1": { + "precision": 0.7058823529411765, + "recall": 0.09063253959928669, + "f1-score": 0.16063958352700566, + "support": 9533.0 + }, + "accuracy": 0.6587034587034587, + "macro avg": { + "precision": 0.6811485404276252, + "recall": 0.5346792292607, + "f1-score": 0.4732218390673721, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6742402758023133, + "recall": 0.6587034587034587, + "f1-score": 0.5605274631908937, + "support": 26455.0 + }, + "cross-entropy": 0.7564828184565525, + "roc-auc": 0.5911993816464689, + "score": 0.5911993816464689 + } + }, + "time": "0:01:33.915789", + "diversity": { + "train": { + "mean_centered_corr": -0.013111491687595844, + "mean_pairwise_disagreement": 0.02585577938919114, + "member_std": 0.02714134193956852 + }, + "val": { + "mean_centered_corr": -0.004636597353965044, + "mean_pairwise_disagreement": 0.08690653790565238, + "member_std": 0.060862503945827484 + }, + "test": { + "mean_centered_corr": 0.0027443149592727423, + "mean_pairwise_disagreement": 0.05465580626870949, + "member_std": 0.05927826836705208 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..a92177d15 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,230 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.001-evaluation/_esam_diagnostics/esam_rho0.001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.782973008757495, + "recall": 0.9834994476056974, + "f1-score": 0.871854546986315, + "support": 84179.0 + }, + "1": { + "precision": 0.6144879267277269, + "recall": 0.08798982592798664, + "f1-score": 0.1539370763080132, + "support": 25162.0 + }, + "accuracy": 0.7774210954719638, + "macro avg": { + "precision": 0.6987304677426109, + "recall": 0.535744636766842, + "f1-score": 0.5128958116471641, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7442005296871277, + "recall": 0.7774210954719638, + "f1-score": 0.7066444300383502, + "support": 109341.0 + }, + "cross-entropy": 0.4979289521091686, + "roc-auc": 0.6847935169951291, + "score": 0.6847935169951291 + }, + "val": { + "0": { + "precision": 0.6776083871263104, + "recall": 0.9537955922154917, + "f1-score": 0.7923235459436311, + "support": 15518.0 + }, + "1": { + "precision": 0.7034739454094293, + "recall": 0.1945556445156125, + "f1-score": 0.3048113968282412, + "support": 8743.0 + }, + "accuracy": 0.6801863072420757, + "macro avg": { + "precision": 0.6905411662678699, + "recall": 0.5741756183655521, + "f1-score": 0.5485674713859361, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6869296259898902, + "recall": 0.6801863072420757, + "f1-score": 0.6166375181741305, + "support": 24261.0 + }, + "cross-entropy": 0.639843807695552, + "roc-auc": 0.641845068859757, + "score": 0.641845068859757 + }, + "test": { + "0": { + "precision": 0.6588995730758489, + "recall": 0.975889374778395, + "f1-score": 0.7866619030606169, + "support": 16922.0 + }, + "1": { + "precision": 0.7068965517241379, + "recall": 0.10322039232140984, + "f1-score": 0.18013729977116705, + "support": 9533.0 + }, + "accuracy": 0.6614250614250614, + "macro avg": { + "precision": 0.6828980623999934, + "recall": 0.5395548835499024, + "f1-score": 0.483399601415892, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6761951768352191, + "recall": 0.6614250614250614, + "f1-score": 0.5681021206694499, + "support": 26455.0 + }, + "cross-entropy": 0.6818687369091303, + "roc-auc": 0.6041622806453656, + "score": 0.6041622806453656 + } + }, + "time": "0:01:33.915789", + "diversity": { + "train": { + "mean_centered_corr": -0.013111491687595844, + "mean_pairwise_disagreement": 0.02585577938919114, + "member_std": 0.02714134193956852 + }, + "val": { + "mean_centered_corr": -0.004636597353965044, + "mean_pairwise_disagreement": 0.08690653790565238, + "member_std": 0.060862503945827484 + }, + "test": { + "mean_centered_corr": 0.0027443149592727423, + "mean_pairwise_disagreement": 0.05465580626870949, + "member_std": 0.05927826836705208 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0, + 25 + ], + "head_selection_time": "0:00:15.827629" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.0025-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.0025-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.0025-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.0025-best-head-evaluation/0/report.json new file mode 100644 index 000000000..32626ae07 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.0025-best-head-evaluation/0/report.json @@ -0,0 +1,229 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.0025-evaluation/_esam_diagnostics/esam_rho0.0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7842672905729285, + "recall": 0.9795911094215897, + "f1-score": 0.8711144446263792, + "support": 84179.0 + }, + "1": { + "precision": 0.5906599952346915, + "recall": 0.09852158016055958, + "f1-score": 0.16887496168125618, + "support": 25162.0 + }, + "accuracy": 0.776835770662423, + "macro avg": { + "precision": 0.68746364290381, + "recall": 0.5390563447910747, + "f1-score": 0.5199947031538177, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7397135845952924, + "recall": 0.776835770662423, + "f1-score": 0.7095122106074367, + "support": 109341.0 + }, + "cross-entropy": 0.49927958217372836, + "roc-auc": 0.6803137982602562, + "score": 0.6803137982602562 + }, + "val": { + "0": { + "precision": 0.6883377360336562, + "recall": 0.906753447609228, + "f1-score": 0.7825917686318131, + "support": 15518.0 + }, + "1": { + "precision": 0.6211050013092433, + "recall": 0.271302756490907, + "f1-score": 0.37764687151727433, + "support": 8743.0 + }, + "accuracy": 0.6777544206751577, + "macro avg": { + "precision": 0.6547213686714497, + "recall": 0.5890281020500675, + "f1-score": 0.5801193200745437, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6641088996421002, + "recall": 0.6777544206751577, + "f1-score": 0.636660717336631, + "support": 24261.0 + }, + "cross-entropy": 0.6314923590147058, + "roc-auc": 0.64045684285539, + "score": 0.64045684285539 + }, + "test": { + "0": { + "precision": 0.6676155391828533, + "recall": 0.9424417917503841, + "f1-score": 0.7815731438372948, + "support": 16922.0 + }, + "1": { + "precision": 0.6205687573042462, + "recall": 0.16710374488618482, + "f1-score": 0.26330578512396696, + "support": 9533.0 + }, + "accuracy": 0.663050463050463, + "macro avg": { + "precision": 0.6440921482435498, + "recall": 0.5547727683182845, + "f1-score": 0.5224394644806309, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6506623367013276, + "recall": 0.663050463050463, + "f1-score": 0.5948166618635978, + "support": 26455.0 + }, + "cross-entropy": 0.6635800829766905, + "roc-auc": 0.6022469172053364, + "score": 0.6022469172053364 + } + }, + "time": "0:01:33.840464", + "diversity": { + "train": { + "mean_centered_corr": -0.013077761046588421, + "mean_pairwise_disagreement": 0.02595400347123574, + "member_std": 0.027166401967406273 + }, + "val": { + "mean_centered_corr": -0.004114337265491486, + "mean_pairwise_disagreement": 0.0868821060217447, + "member_std": 0.060897231101989746 + }, + "test": { + "mean_centered_corr": 0.003803197992965579, + "mean_pairwise_disagreement": 0.05452624968754001, + "member_std": 0.05941416695713997 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0 + ], + "head_selection_time": "0:00:04.729629" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.0025-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.0025-evaluation/0.toml new file mode 100644 index 000000000..361754c34 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.0025-evaluation/0.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.0025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.0025-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.0025-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.0025-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.0025-evaluation/0/report.json new file mode 100644 index 000000000..4a2c19b44 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.0025-evaluation/0/report.json @@ -0,0 +1,225 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.0025-evaluation/_esam_diagnostics/esam_rho0.0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7862332330898849, + "recall": 0.9783200085532021, + "f1-score": 0.8718214731849845, + "support": 84179.0 + }, + "1": { + "precision": 0.6029155787641427, + "recall": 0.11012638105079088, + "f1-score": 0.18623563411519592, + "support": 25162.0 + }, + "accuracy": 0.7785277251900019, + "macro avg": { + "precision": 0.6945744059270138, + "recall": 0.5442231948019965, + "f1-score": 0.5290285536500902, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7440474215631535, + "recall": 0.7785277251900019, + "f1-score": 0.7140516440936644, + "support": 109341.0 + }, + "cross-entropy": 0.4923044781282782, + "roc-auc": 0.6989427314976193, + "score": 0.6989427314976193 + }, + "val": { + "0": { + "precision": 0.6752206752206752, + "recall": 0.9563088026807578, + "f1-score": 0.7915510987838703, + "support": 15518.0 + }, + "1": { + "precision": 0.7030223390275953, + "recall": 0.18357543177399063, + "f1-score": 0.29113005623072735, + "support": 8743.0 + }, + "accuracy": 0.6778368575079345, + "macro avg": { + "precision": 0.6891215071241352, + "recall": 0.5699421172273742, + "f1-score": 0.5413405775072988, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6852396334937845, + "recall": 0.6778368575079345, + "f1-score": 0.6112130593361094, + "support": 24261.0 + }, + "cross-entropy": 0.6870530045533841, + "roc-auc": 0.6227333347907498, + "score": 0.6227333347907498 + }, + "test": { + "0": { + "precision": 0.6564531472966545, + "recall": 0.9786668242524524, + "f1-score": 0.7858125741399763, + "support": 16922.0 + }, + "1": { + "precision": 0.7057864710676447, + "recall": 0.09084233714465541, + "f1-score": 0.16096654275092936, + "support": 9533.0 + }, + "accuracy": 0.6587412587412588, + "macro avg": { + "precision": 0.6811198091821495, + "recall": 0.534754580698554, + "f1-score": 0.47338955844545283, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6742303000280417, + "recall": 0.6587412587412588, + "f1-score": 0.5606507061667468, + "support": 26455.0 + }, + "cross-entropy": 0.7560531851992458, + "roc-auc": 0.5911601949314514, + "score": 0.5911601949314514 + } + }, + "time": "0:01:33.840464", + "diversity": { + "train": { + "mean_centered_corr": -0.013077761046588421, + "mean_pairwise_disagreement": 0.02595400347123574, + "member_std": 0.027166401967406273 + }, + "val": { + "mean_centered_corr": -0.004114337265491486, + "mean_pairwise_disagreement": 0.0868821060217447, + "member_std": 0.060897231101989746 + }, + "test": { + "mean_centered_corr": 0.003803197992965579, + "mean_pairwise_disagreement": 0.05452624968754001, + "member_std": 0.05941416695713997 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.0025-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.0025-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.0025-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.0025-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..42478e4a6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.0025-greedy-heads-evaluation/0/report.json @@ -0,0 +1,230 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.0025-evaluation/_esam_diagnostics/esam_rho0.0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7829495484420067, + "recall": 0.9835350859478017, + "f1-score": 0.8718540047597987, + "support": 84179.0 + }, + "1": { + "precision": 0.6145717463848721, + "recall": 0.087830856052778, + "f1-score": 0.15369636275123444, + "support": 25162.0 + }, + "accuracy": 0.7774119497718148, + "macro avg": { + "precision": 0.6987606474134394, + "recall": 0.5356829710002898, + "f1-score": 0.5127751837555166, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7442017570795569, + "recall": 0.7774119497718148, + "f1-score": 0.7065886185988938, + "support": 109341.0 + }, + "cross-entropy": 0.4979388170923032, + "roc-auc": 0.6847377557794279, + "score": 0.6847377557794279 + }, + "val": { + "0": { + "precision": 0.6775373919407218, + "recall": 0.9545688877432659, + "f1-score": 0.7925416655520184, + "support": 15518.0 + }, + "1": { + "precision": 0.7060050041701418, + "recall": 0.193640626787144, + "f1-score": 0.3039224486132304, + "support": 8743.0 + }, + "accuracy": 0.6803511809076296, + "macro avg": { + "precision": 0.6917711980554317, + "recall": 0.5741047572652049, + "f1-score": 0.5482320570826245, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6877963397879588, + "recall": 0.6803511809076296, + "f1-score": 0.6164566808565887, + "support": 24261.0 + }, + "cross-entropy": 0.6400045165065813, + "roc-auc": 0.641480466607742, + "score": 0.641480466607742 + }, + "test": { + "0": { + "precision": 0.6585200542221513, + "recall": 0.9760666587873774, + "f1-score": 0.7864489096276545, + "support": 16922.0 + }, + "1": { + "precision": 0.7050254916241806, + "recall": 0.10154201195846009, + "f1-score": 0.1775169631395562, + "support": 9533.0 + }, + "accuracy": 0.6609336609336609, + "macro avg": { + "precision": 0.681772772923166, + "recall": 0.5388043353729187, + "f1-score": 0.48198293638360534, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6752781844339656, + "recall": 0.6609336609336609, + "f1-score": 0.567021646506466, + "support": 26455.0 + }, + "cross-entropy": 0.6817906405461313, + "roc-auc": 0.6039784195416061, + "score": 0.6039784195416061 + } + }, + "time": "0:01:33.840464", + "diversity": { + "train": { + "mean_centered_corr": -0.013077761046588421, + "mean_pairwise_disagreement": 0.02595400347123574, + "member_std": 0.027166401967406273 + }, + "val": { + "mean_centered_corr": -0.004114337265491486, + "mean_pairwise_disagreement": 0.0868821060217447, + "member_std": 0.060897231101989746 + }, + "test": { + "mean_centered_corr": 0.003803197992965579, + "mean_pairwise_disagreement": 0.05452624968754001, + "member_std": 0.05941416695713997 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0, + 25 + ], + "head_selection_time": "0:00:15.726306" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..aa7be9cc7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.005-best-head-evaluation/0/report.json @@ -0,0 +1,229 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.005-evaluation/_esam_diagnostics/esam_rho0.005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7843107416636549, + "recall": 0.9796267477636941, + "f1-score": 0.8711553393442882, + "support": 84179.0 + }, + "1": { + "precision": 0.5915694212907835, + "recall": 0.09872029250457039, + "f1-score": 0.1692040461837131, + "support": 25162.0 + }, + "accuracy": 0.7769089362636157, + "macro avg": { + "precision": 0.6879400814772192, + "recall": 0.5391735201341322, + "f1-score": 0.5201796927640007, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7399563174017385, + "recall": 0.7769089362636157, + "f1-score": 0.709619424742205, + "support": 109341.0 + }, + "cross-entropy": 0.4992874898761024, + "roc-auc": 0.680286897888579, + "score": 0.680286897888579 + }, + "val": { + "0": { + "precision": 0.6876976212482366, + "recall": 0.9110065730119861, + "f1-score": 0.7837560637560638, + "support": 15518.0 + }, + "1": { + "precision": 0.6271598272138229, + "recall": 0.2656982729040375, + "f1-score": 0.3732626335663212, + "support": 8743.0 + }, + "accuracy": 0.6784551337537612, + "macro avg": { + "precision": 0.6574287242310297, + "recall": 0.5883524229580118, + "f1-score": 0.5785093486611925, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6658814581369519, + "recall": 0.6784551337537612, + "f1-score": 0.6358254730900188, + "support": 24261.0 + }, + "cross-entropy": 0.6313069278816857, + "roc-auc": 0.6406711951042247, + "score": 0.6406711951042247 + }, + "test": { + "0": { + "precision": 0.6672369189685388, + "recall": 0.9449828625457983, + "f1-score": 0.7821854822930934, + "support": 16922.0 + }, + "1": { + "precision": 0.6259541984732825, + "recall": 0.16343228784223224, + "f1-score": 0.2591914822824821, + "support": 9533.0 + }, + "accuracy": 0.6633528633528634, + "macro avg": { + "precision": 0.6465955587209107, + "recall": 0.5542075751940153, + "f1-score": 0.5206884822877877, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6523607830970105, + "recall": 0.6633528633528634, + "f1-score": 0.5937257657139531, + "support": 26455.0 + }, + "cross-entropy": 0.6637267283308612, + "roc-auc": 0.6023965414622969, + "score": 0.6023965414622969 + } + }, + "time": "0:01:33.976122", + "diversity": { + "train": { + "mean_centered_corr": -0.013189224526286125, + "mean_pairwise_disagreement": 0.02590141569537856, + "member_std": 0.027156289666891098 + }, + "val": { + "mean_centered_corr": -0.003924679942429066, + "mean_pairwise_disagreement": 0.08662631915552772, + "member_std": 0.060857683420181274 + }, + "test": { + "mean_centered_corr": 0.004035224672406912, + "mean_pairwise_disagreement": 0.054524420653452914, + "member_std": 0.059433531016111374 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0 + ], + "head_selection_time": "0:00:04.881229" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.005-evaluation/0.toml new file mode 100644 index 000000000..6c0027518 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.005-evaluation/0.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.005-evaluation/0/report.json new file mode 100644 index 000000000..5458b2eef --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.005-evaluation/0/report.json @@ -0,0 +1,225 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.005-evaluation/_esam_diagnostics/esam_rho0.005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.786244164176397, + "recall": 0.9782962496584658, + "f1-score": 0.8718187592631802, + "support": 84179.0 + }, + "1": { + "precision": 0.6028260869565217, + "recall": 0.1102058659883952, + "f1-score": 0.18634500369598817, + "support": 25162.0 + }, + "accuracy": 0.7785277251900019, + "macro avg": { + "precision": 0.6945351255664594, + "recall": 0.5442510578234305, + "f1-score": 0.5290818814795841, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7440352429208157, + "recall": 0.7785277251900019, + "f1-score": 0.7140747232878215, + "support": 109341.0 + }, + "cross-entropy": 0.49232123937555056, + "roc-auc": 0.6989090994233629, + "score": 0.6989090994233629 + }, + "val": { + "0": { + "precision": 0.6752969013059107, + "recall": 0.956373243974739, + "f1-score": 0.7916255500733431, + "support": 15518.0 + }, + "1": { + "precision": 0.7035901926444834, + "recall": 0.18380418620610775, + "f1-score": 0.29146640065294277, + "support": 8743.0 + }, + "accuracy": 0.6779605127570999, + "macro avg": { + "precision": 0.689443546975197, + "recall": 0.5700887150904234, + "f1-score": 0.541545975363143, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6854930286779539, + "recall": 0.6779605127570999, + "f1-score": 0.611381889738544, + "support": 24261.0 + }, + "cross-entropy": 0.6870363427483223, + "roc-auc": 0.6227207494642631, + "score": 0.6227207494642631 + }, + "test": { + "0": { + "precision": 0.6564519326065411, + "recall": 0.9785486349131308, + "f1-score": 0.7857736019170997, + "support": 16922.0 + }, + "1": { + "precision": 0.7048780487804878, + "recall": 0.09094723591733976, + "f1-score": 0.1611074979095048, + "support": 9533.0 + }, + "accuracy": 0.6587034587034587, + "macro avg": { + "precision": 0.6806649906935145, + "recall": 0.5347479354152352, + "f1-score": 0.4734405499133022, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6739021751121632, + "recall": 0.6587034587034587, + "f1-score": 0.5606765703728017, + "support": 26455.0 + }, + "cross-entropy": 0.7560552620145878, + "roc-auc": 0.5911612394559284, + "score": 0.5911612394559284 + } + }, + "time": "0:01:33.976122", + "diversity": { + "train": { + "mean_centered_corr": -0.013189224526286125, + "mean_pairwise_disagreement": 0.02590141569537856, + "member_std": 0.027156289666891098 + }, + "val": { + "mean_centered_corr": -0.003924679942429066, + "mean_pairwise_disagreement": 0.08662631915552772, + "member_std": 0.060857683420181274 + }, + "test": { + "mean_centered_corr": 0.004035224672406912, + "mean_pairwise_disagreement": 0.054524420653452914, + "member_std": 0.059433531016111374 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..8ff9400b7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,230 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.005-evaluation/_esam_diagnostics/esam_rho0.005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7830293972986266, + "recall": 0.9834400503688568, + "f1-score": 0.8718661632517654, + "support": 84179.0 + }, + "1": { + "precision": 0.6145977329278407, + "recall": 0.0883475081472061, + "f1-score": 0.1544876472427812, + "support": 25162.0 + }, + "accuracy": 0.7774576782725602, + "macro avg": { + "precision": 0.6988135651132337, + "recall": 0.5358937792580315, + "f1-score": 0.5131769052472732, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7442692109193387, + "recall": 0.7774576782725602, + "f1-score": 0.706780072765872, + "support": 109341.0 + }, + "cross-entropy": 0.4979356402925865, + "roc-auc": 0.6847042861139584, + "score": 0.6847042861139584 + }, + "val": { + "0": { + "precision": 0.6772467413674823, + "recall": 0.95424668127336, + "f1-score": 0.7922317631008747, + "support": 15518.0 + }, + "1": { + "precision": 0.7036727879799666, + "recall": 0.19283998627473406, + "f1-score": 0.30272017236735793, + "support": 8743.0 + }, + "accuracy": 0.6798565599109683, + "macro avg": { + "precision": 0.6904597646737245, + "recall": 0.573543333774047, + "f1-score": 0.5474759677341163, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6867699648757033, + "recall": 0.6798565599109683, + "f1-score": 0.6158251913279412, + "support": 24261.0 + }, + "cross-entropy": 0.6400161363311784, + "roc-auc": 0.6415752785241469, + "score": 0.6415752785241469 + }, + "test": { + "0": { + "precision": 0.6584811640422563, + "recall": 0.9761257534570382, + "f1-score": 0.7864403551788988, + "support": 16922.0 + }, + "1": { + "precision": 0.7051094890510949, + "recall": 0.10133221441309137, + "f1-score": 0.17719893607264056, + "support": 9533.0 + }, + "accuracy": 0.6608958608958609, + "macro avg": { + "precision": 0.6817953265466756, + "recall": 0.5387289839350649, + "f1-score": 0.4818196456257697, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6752835765279588, + "recall": 0.6608958608958609, + "f1-score": 0.5669015742928675, + "support": 26455.0 + }, + "cross-entropy": 0.6818542221067515, + "roc-auc": 0.6040898334194843, + "score": 0.6040898334194843 + } + }, + "time": "0:01:33.976122", + "diversity": { + "train": { + "mean_centered_corr": -0.013189224526286125, + "mean_pairwise_disagreement": 0.02590141569537856, + "member_std": 0.027156289666891098 + }, + "val": { + "mean_centered_corr": -0.003924679942429066, + "mean_pairwise_disagreement": 0.08662631915552772, + "member_std": 0.060857683420181274 + }, + "test": { + "mean_centered_corr": 0.004035224672406912, + "mean_pairwise_disagreement": 0.054524420653452914, + "member_std": 0.059433531016111374 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0, + 25 + ], + "head_selection_time": "0:00:16.191151" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..65e0a5aa0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.01-best-head-evaluation/0/report.json @@ -0,0 +1,229 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.01-evaluation/_esam_diagnostics/esam_rho0.01-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7841306662610283, + "recall": 0.9797930600268475, + "f1-score": 0.8711099845270723, + "support": 84179.0 + }, + "1": { + "precision": 0.5908106807794082, + "recall": 0.09760750337810985, + "f1-score": 0.1675364098366247, + "support": 25162.0 + }, + "accuracy": 0.7767808964615286, + "macro avg": { + "precision": 0.6874706735202183, + "recall": 0.5387002817024786, + "f1-score": 0.5193231971818485, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7396430772076218, + "recall": 0.7767808964615286, + "f1-score": 0.7092007438363795, + "support": 109341.0 + }, + "cross-entropy": 0.4994019756555021, + "roc-auc": 0.6800035420034479, + "score": 0.6800035420034479 + }, + "val": { + "0": { + "precision": 0.6883509833585476, + "recall": 0.9089444516045883, + "f1-score": 0.783415257296787, + "support": 15518.0 + }, + "1": { + "precision": 0.6251989389920425, + "recall": 0.2695870982500286, + "f1-score": 0.376728202669224, + "support": 8743.0 + }, + "accuracy": 0.6785375705865381, + "macro avg": { + "precision": 0.6567749611752951, + "recall": 0.5892657749273085, + "f1-score": 0.5800717299830055, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.665592716020171, + "recall": 0.6785375705865381, + "f1-score": 0.6368563801437931, + "support": 24261.0 + }, + "cross-entropy": 0.6311702903663475, + "roc-auc": 0.6407133660825518, + "score": 0.6407133660825518 + }, + "test": { + "0": { + "precision": 0.6672656609135359, + "recall": 0.9435645904739393, + "f1-score": 0.7817189297691611, + "support": 16922.0 + }, + "1": { + "precision": 0.6219319081551861, + "recall": 0.1647959718871289, + "f1-score": 0.26055228460071317, + "support": 9533.0 + }, + "accuracy": 0.6629370629370629, + "macro avg": { + "precision": 0.644598784534361, + "recall": 0.5541802811805341, + "f1-score": 0.5211356071849371, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6509297446389054, + "recall": 0.6629370629370629, + "f1-score": 0.593917696414755, + "support": 26455.0 + }, + "cross-entropy": 0.6633037626379901, + "roc-auc": 0.6023744607727624, + "score": 0.6023744607727624 + } + }, + "time": "0:01:33.760661", + "diversity": { + "train": { + "mean_centered_corr": -0.012859287671744823, + "mean_pairwise_disagreement": 0.02587178436445202, + "member_std": 0.027145707979798317 + }, + "val": { + "mean_centered_corr": -0.003945968113839626, + "mean_pairwise_disagreement": 0.08649867502735706, + "member_std": 0.061113499104976654 + }, + "test": { + "mean_centered_corr": 0.003282116027548909, + "mean_pairwise_disagreement": 0.05447816133300004, + "member_std": 0.05972920358181 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0 + ], + "head_selection_time": "0:00:04.815423" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.01-evaluation/0.toml new file mode 100644 index 000000000..69b72ad9f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.01-evaluation/0.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.01 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.01-evaluation/0/report.json new file mode 100644 index 000000000..817f2bf62 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.01-evaluation/0/report.json @@ -0,0 +1,225 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.01-evaluation/_esam_diagnostics/esam_rho0.01-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7860374117197938, + "recall": 0.9784031646847788, + "f1-score": 0.8717340798797623, + "support": 84179.0 + }, + "1": { + "precision": 0.6014032010524007, + "recall": 0.10901359192433034, + "f1-score": 0.18457087104262693, + "support": 25162.0 + }, + "accuracy": 0.7783356654868714, + "macro avg": { + "precision": 0.6937203063860973, + "recall": 0.5437083783045545, + "f1-score": 0.5281524754611946, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.743548628840426, + "recall": 0.7783356654868714, + "f1-score": 0.7136012599790845, + "support": 109341.0 + }, + "cross-entropy": 0.49236964035190794, + "roc-auc": 0.6988119397829878, + "score": 0.6988119397829878 + }, + "val": { + "0": { + "precision": 0.67549849767823, + "recall": 0.9561799200927955, + "f1-score": 0.7916977910575179, + "support": 15518.0 + }, + "1": { + "precision": 0.7037037037037037, + "recall": 0.18471920393457622, + "f1-score": 0.29262547562964303, + "support": 8743.0 + }, + "accuracy": 0.6781666048390421, + "macro avg": { + "precision": 0.6896011006909668, + "recall": 0.5704495620136859, + "f1-score": 0.5421616333435805, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6856628815156942, + "recall": 0.6781666048390421, + "f1-score": 0.6118457959301155, + "support": 24261.0 + }, + "cross-entropy": 0.686500348338853, + "roc-auc": 0.6227573335158101, + "score": 0.6227573335158101 + }, + "test": { + "0": { + "precision": 0.6564879441624365, + "recall": 0.9782531615648269, + "f1-score": 0.7857041150505482, + "support": 16922.0 + }, + "1": { + "precision": 0.7029862792574657, + "recall": 0.0913668310080772, + "f1-score": 0.1617155588562941, + "support": 9533.0 + }, + "accuracy": 0.6586656586656586, + "macro avg": { + "precision": 0.6797371117099511, + "recall": 0.5348099962864521, + "f1-score": 0.4737098369534211, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.67324351507383, + "recall": 0.6586656586656586, + "f1-score": 0.5608512363433161, + "support": 26455.0 + }, + "cross-entropy": 0.7551689618359527, + "roc-auc": 0.5911827002496308, + "score": 0.5911827002496308 + } + }, + "time": "0:01:33.760661", + "diversity": { + "train": { + "mean_centered_corr": -0.012859287671744823, + "mean_pairwise_disagreement": 0.02587178436445202, + "member_std": 0.027145707979798317 + }, + "val": { + "mean_centered_corr": -0.003945968113839626, + "mean_pairwise_disagreement": 0.08649867502735706, + "member_std": 0.061113499104976654 + }, + "test": { + "mean_centered_corr": 0.003282116027548909, + "mean_pairwise_disagreement": 0.05447816133300004, + "member_std": 0.05972920358181 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..357dac6a7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,230 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.01-evaluation/_esam_diagnostics/esam_rho0.01-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7828372603154192, + "recall": 0.983570724289906, + "f1-score": 0.8717983816197491, + "support": 84179.0 + }, + "1": { + "precision": 0.6133631534805704, + "recall": 0.0871949765519434, + "f1-score": 0.15268450537596995, + "support": 25162.0 + }, + "accuracy": 0.7772930556698768, + "macro avg": { + "precision": 0.6981002068979948, + "recall": 0.5353828504209247, + "f1-score": 0.5122414434978595, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7438371827948325, + "recall": 0.7772930556698768, + "f1-score": 0.7063129429092382, + "support": 109341.0 + }, + "cross-entropy": 0.49804174438759463, + "roc-auc": 0.6845043493304455, + "score": 0.6845043493304455 + }, + "val": { + "0": { + "precision": 0.6775611095852788, + "recall": 0.9538600335094729, + "f1-score": 0.7923134568033401, + "support": 15518.0 + }, + "1": { + "precision": 0.7035196687370601, + "recall": 0.19432689008349538, + "f1-score": 0.3045348628786521, + "support": 8743.0 + }, + "accuracy": 0.6801450888256874, + "macro avg": { + "precision": 0.6905403891611694, + "recall": 0.5740934617964841, + "f1-score": 0.5484241598409961, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6869158634150477, + "recall": 0.6801450888256874, + "f1-score": 0.616531409621297, + "support": 24261.0 + }, + "cross-entropy": 0.6396176757776643, + "roc-auc": 0.6415860912175324, + "score": 0.6415860912175324 + }, + "test": { + "0": { + "precision": 0.6586747516655364, + "recall": 0.9757120907694126, + "f1-score": 0.7864440686846556, + "support": 16922.0 + }, + "1": { + "precision": 0.7038904899135446, + "recall": 0.10248610091261932, + "f1-score": 0.17892134419924915, + "support": 9533.0 + }, + "accuracy": 0.661047061047061, + "macro avg": { + "precision": 0.6812826207895405, + "recall": 0.5390990958410159, + "f1-score": 0.4826827064419523, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6749681416756766, + "recall": 0.661047061047061, + "f1-score": 0.5675246155559699, + "support": 26455.0 + }, + "cross-entropy": 0.6812321989565008, + "roc-auc": 0.6040412707800087, + "score": 0.6040412707800087 + } + }, + "time": "0:01:33.760661", + "diversity": { + "train": { + "mean_centered_corr": -0.012859287671744823, + "mean_pairwise_disagreement": 0.02587178436445202, + "member_std": 0.027145707979798317 + }, + "val": { + "mean_centered_corr": -0.003945968113839626, + "mean_pairwise_disagreement": 0.08649867502735706, + "member_std": 0.061113499104976654 + }, + "test": { + "mean_centered_corr": 0.003282116027548909, + "mean_pairwise_disagreement": 0.05447816133300004, + "member_std": 0.05972920358181 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0, + 25 + ], + "head_selection_time": "0:00:15.930728" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_cf_fisd-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_cf_fisd-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_cf_fisd-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_cf_fisd-best-head-evaluation/0/report.json new file mode 100644 index 000000000..8fc29a25e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_cf_fisd-best-head-evaluation/0/report.json @@ -0,0 +1,426 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.887499988079071, + "min_features_per_member": 92.0, + "mean_features_per_member": 97.625, + "max_features_per_member": 103.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.399999618530273, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.797659951359852, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7889205040357703, + "recall": 0.9683650316587272, + "f1-score": 0.8694808138449641, + "support": 84179.0 + }, + "1": { + "precision": 0.5572734829592685, + "recall": 0.13321675542484698, + "f1-score": 0.21503031080604293, + "support": 25162.0 + }, + "accuracy": 0.7761772802516896, + "macro avg": { + "precision": 0.6730969934975194, + "recall": 0.5507908935417871, + "f1-score": 0.5422555623255035, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7356129401363461, + "recall": 0.7761772802516896, + "f1-score": 0.7188759761585946, + "support": 109341.0 + }, + "cross-entropy": 0.4988956148214662, + "roc-auc": 0.68250001764071, + "score": 0.68250001764071 + }, + "val": { + "0": { + "precision": 0.6950127877237852, + "recall": 0.910619925248099, + "f1-score": 0.7883403068340307, + "support": 15518.0 + }, + "1": { + "precision": 0.6469839653855943, + "recall": 0.2907468832208624, + "f1-score": 0.401199494949495, + "support": 8743.0 + }, + "accuracy": 0.6872346564444994, + "macro avg": { + "precision": 0.6709983765546897, + "recall": 0.6006834042344806, + "f1-score": 0.5947699008917628, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6777045154471766, + "recall": 0.6872346564444994, + "f1-score": 0.6488253602816835, + "support": 24261.0 + }, + "cross-entropy": 0.6245548911838499, + "roc-auc": 0.6477890245840551, + "score": 0.6477890245840551 + }, + "test": { + "0": { + "precision": 0.6679904029122198, + "recall": 0.9542607256825434, + "f1-score": 0.7858672376873662, + "support": 16922.0 + }, + "1": { + "precision": 0.6606751424813678, + "recall": 0.1580824504353299, + "f1-score": 0.25512104283054005, + "support": 9533.0 + }, + "accuracy": 0.6673596673596673, + "macro avg": { + "precision": 0.6643327726967938, + "recall": 0.5561715880589366, + "f1-score": 0.5204941402589531, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6653543651996018, + "recall": 0.6673596673596673, + "f1-score": 0.5946140350576129, + "support": 26455.0 + }, + "cross-entropy": 0.6584188876218103, + "roc-auc": 0.6045088303107439, + "score": 0.6045088303107439 + } + }, + "time": "0:01:18.654365", + "diversity": { + "train": { + "mean_centered_corr": -0.01063462533056736, + "mean_pairwise_disagreement": 0.026665487313881315, + "member_std": 0.029800426214933395 + }, + "val": { + "mean_centered_corr": -0.013281173072755337, + "mean_pairwise_disagreement": 0.09412748922670262, + "member_std": 0.06749773025512695 + }, + "test": { + "mean_centered_corr": -0.011601177044212818, + "mean_pairwise_disagreement": 0.059119563958273634, + "member_std": 0.06550896167755127 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 22 + ], + "head_selection_time": "0:00:04.756683" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_cf_fisd-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_cf_fisd-evaluation/0.toml new file mode 100644 index 000000000..cd8d74155 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_cf_fisd-evaluation/0.toml @@ -0,0 +1,57 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 47 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "ecom-offers" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_cf_fisd-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_cf_fisd-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_cf_fisd-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_cf_fisd-evaluation/0/report.json new file mode 100644 index 000000000..5e1e2aae3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_cf_fisd-evaluation/0/report.json @@ -0,0 +1,422 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.887499988079071, + "min_features_per_member": 92.0, + "mean_features_per_member": 97.625, + "max_features_per_member": 103.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.399999618530273, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.797659951359852, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7842480378851866, + "recall": 0.9816818921583768, + "f1-score": 0.8719282511210762, + "support": 84179.0 + }, + "1": { + "precision": 0.6115869017632242, + "recall": 0.09649471425164931, + "f1-score": 0.1666895510091995, + "support": 25162.0 + }, + "accuracy": 0.7779789831810574, + "macro avg": { + "precision": 0.6979174698242054, + "recall": 0.539088303205013, + "f1-score": 0.5193089010651378, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7445145480954387, + "recall": 0.7779789831810574, + "f1-score": 0.7096358249294825, + "support": 109341.0 + }, + "cross-entropy": 0.4927771605769271, + "roc-auc": 0.6981357694476362, + "score": 0.6981357694476362 + }, + "val": { + "0": { + "precision": 0.6767607563716087, + "recall": 0.9548266529191907, + "f1-score": 0.7920987918314979, + "support": 15518.0 + }, + "1": { + "precision": 0.7038445289395859, + "recall": 0.19055244195356286, + "f1-score": 0.29990999099909993, + "support": 8743.0 + }, + "accuracy": 0.6794031573306953, + "macro avg": { + "precision": 0.6903026426555974, + "recall": 0.5726895474363768, + "f1-score": 0.5460043914152989, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6865210063020248, + "recall": 0.6794031573306953, + "f1-score": 0.6147274268557072, + "support": 24261.0 + }, + "cross-entropy": 0.673915591317614, + "roc-auc": 0.6250550271749447, + "score": 0.6250550271749447 + }, + "test": { + "0": { + "precision": 0.657534791252485, + "recall": 0.9772485521805933, + "f1-score": 0.7861285415478227, + "support": 16922.0 + }, + "1": { + "precision": 0.7049808429118773, + "recall": 0.09650687086961082, + "f1-score": 0.16977302085255583, + "support": 9533.0 + }, + "accuracy": 0.6598752598752599, + "macro avg": { + "precision": 0.6812578170821812, + "recall": 0.536877711525102, + "f1-score": 0.4779507812001893, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6746318697052912, + "recall": 0.6598752598752599, + "f1-score": 0.5640262100873057, + "support": 26455.0 + }, + "cross-entropy": 0.735759481729751, + "roc-auc": 0.592670158895295, + "score": 0.592670158895295 + } + }, + "time": "0:01:18.654365", + "diversity": { + "train": { + "mean_centered_corr": -0.01063462533056736, + "mean_pairwise_disagreement": 0.026665487313881315, + "member_std": 0.029800426214933395 + }, + "val": { + "mean_centered_corr": -0.013281173072755337, + "mean_pairwise_disagreement": 0.09412748922670262, + "member_std": 0.06749773025512695 + }, + "test": { + "mean_centered_corr": -0.011601177044212818, + "mean_pairwise_disagreement": 0.059119563958273634, + "member_std": 0.06550896167755127 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_cf_fisd-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_cf_fisd-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_cf_fisd-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_cf_fisd-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..5f7214ce5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_cf_fisd-greedy-heads-evaluation/0/report.json @@ -0,0 +1,428 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.887499988079071, + "min_features_per_member": 92.0, + "mean_features_per_member": 97.625, + "max_features_per_member": 103.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.399999618530273, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.797659951359852, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7909659567005086, + "recall": 0.9643735373430428, + "f1-score": 0.8691043985161632, + "support": 84179.0 + }, + "1": { + "precision": 0.5528552258834054, + "recall": 0.14736507431841667, + "f1-score": 0.23270262637672973, + "support": 25162.0 + }, + "accuracy": 0.7763601942546712, + "macro avg": { + "precision": 0.671910591291957, + "recall": 0.5558693058307297, + "f1-score": 0.5509035124464464, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7361709373681451, + "recall": 0.7763601942546712, + "f1-score": 0.7226530089132472, + "support": 109341.0 + }, + "cross-entropy": 0.5000394688239905, + "roc-auc": 0.6809790060024956, + "score": 0.6809790060024956 + }, + "val": { + "0": { + "precision": 0.7075808427894092, + "recall": 0.8559092666580745, + "f1-score": 0.7747090903788387, + "support": 15518.0 + }, + "1": { + "precision": 0.5927140255009108, + "recall": 0.3721834610545579, + "f1-score": 0.4572472423241762, + "support": 8743.0 + }, + "accuracy": 0.6815877333992828, + "macro avg": { + "precision": 0.6501474341451601, + "recall": 0.6140463638563163, + "f1-score": 0.6159781663515075, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6661859875256797, + "recall": 0.6815877333992828, + "f1-score": 0.6603045341964096, + "support": 24261.0 + }, + "cross-entropy": 0.6203779736299148, + "roc-auc": 0.6506783465179156, + "score": 0.6506783465179156 + }, + "test": { + "0": { + "precision": 0.6800559465011583, + "recall": 0.9194539652523342, + "f1-score": 0.7818396522700435, + "support": 16922.0 + }, + "1": { + "precision": 0.618847874720358, + "recall": 0.23214098395048777, + "f1-score": 0.3376306354412999, + "support": 9533.0 + }, + "accuracy": 0.6717822717822718, + "macro avg": { + "precision": 0.6494519106107581, + "recall": 0.575797474601411, + "f1-score": 0.5597351438556717, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6579997549197419, + "recall": 0.6717822717822718, + "f1-score": 0.6217699279295251, + "support": 26455.0 + }, + "cross-entropy": 0.6443009499315193, + "roc-auc": 0.6065618416202598, + "score": 0.6065618416202598 + } + }, + "time": "0:01:18.654365", + "diversity": { + "train": { + "mean_centered_corr": -0.01063462533056736, + "mean_pairwise_disagreement": 0.026665487313881315, + "member_std": 0.029800426214933395 + }, + "val": { + "mean_centered_corr": -0.013281173072755337, + "mean_pairwise_disagreement": 0.09412748922670262, + "member_std": 0.06749773025512695 + }, + "test": { + "mean_centered_corr": -0.011601177044212818, + "mean_pairwise_disagreement": 0.059119563958273634, + "member_std": 0.06550896167755127 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 22, + 24, + 24 + ], + "head_selection_time": "0:00:20.435848" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.7-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.7-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.7-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.7-best-head-evaluation/0/report.json new file mode 100644 index 000000000..85c7d6ec0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.7-best-head-evaluation/0/report.json @@ -0,0 +1,255 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.6894886493682861, + "min_features_per_member": 67.0, + "mean_features_per_member": 75.84375, + "max_features_per_member": 84.0, + "min_members_per_feature": 14.0, + "mean_members_per_feature": 22.063636779785156, + "max_members_per_feature": 31.0, + "mean_pairwise_jaccard": 0.5278457608823273, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 428, + "metrics": { + "train": { + "0": { + "precision": 0.7894966160281175, + "recall": 0.9686501383955618, + "f1-score": 0.8699455883921904, + "support": 84179.0 + }, + "1": { + "precision": 0.5645214521452145, + "recall": 0.13595898577219617, + "f1-score": 0.21914034975337904, + "support": 25162.0 + }, + "accuracy": 0.7770278303655537, + "macro avg": { + "precision": 0.677009034086666, + "recall": 0.552304562083879, + "f1-score": 0.5445429690727848, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7377244073084094, + "recall": 0.7770278303655537, + "f1-score": 0.7201796139212255, + "support": 109341.0 + }, + "cross-entropy": 0.5002029863194409, + "roc-auc": 0.6821879786641952, + "score": 0.6821879786641952 + }, + "val": { + "0": { + "precision": 0.6961402811584124, + "recall": 0.9030802938523006, + "f1-score": 0.7862212123762238, + "support": 15518.0 + }, + "1": { + "precision": 0.6358353510895883, + "recall": 0.30035456936978155, + "f1-score": 0.4079857065175173, + "support": 8743.0 + }, + "accuracy": 0.6858744487036808, + "macro avg": { + "precision": 0.6659878161240004, + "recall": 0.6017174316110411, + "f1-score": 0.5971034594468705, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6744080358432264, + "recall": 0.6858744487036808, + "f1-score": 0.6499154942391862, + "support": 24261.0 + }, + "cross-entropy": 0.6154470541736884, + "roc-auc": 0.6547032702847417, + "score": 0.6547032702847417 + }, + "test": { + "0": { + "precision": 0.6733296707940667, + "recall": 0.9415553717054722, + "f1-score": 0.7851669335961562, + "support": 16922.0 + }, + "1": { + "precision": 0.6457736389684814, + "recall": 0.18913248714990036, + "f1-score": 0.2925760649087221, + "support": 9533.0 + }, + "accuracy": 0.6704214704214704, + "macro avg": { + "precision": 0.6595516548812741, + "recall": 0.5653439294276863, + "f1-score": 0.5388714992524392, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6633999164416454, + "recall": 0.6704214704214704, + "f1-score": 0.6076629172968816, + "support": 26455.0 + }, + "cross-entropy": 0.638677137478482, + "roc-auc": 0.6118564029158264, + "score": 0.6118564029158264 + } + }, + "time": "0:01:10.628940", + "diversity": { + "train": { + "mean_centered_corr": -0.015634549781680107, + "mean_pairwise_disagreement": 0.03130809179096706, + "member_std": 0.0355970673263073 + }, + "val": { + "mean_centered_corr": -0.0187542624771595, + "mean_pairwise_disagreement": 0.12205770312369114, + "member_std": 0.07598911970853806 + }, + "test": { + "mean_centered_corr": -0.018549803644418716, + "mean_pairwise_disagreement": 0.08149855811146134, + "member_std": 0.0770004391670227 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 16 + ], + "head_selection_time": "0:00:04.834038" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.7-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.7-evaluation/0.toml new file mode 100644 index 000000000..5319a963b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.7-evaluation/0.toml @@ -0,0 +1,46 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.7 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.7-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.7-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.7-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.7-evaluation/0/report.json new file mode 100644 index 000000000..d3d96f1bb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.7-evaluation/0/report.json @@ -0,0 +1,251 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.6894886493682861, + "min_features_per_member": 67.0, + "mean_features_per_member": 75.84375, + "max_features_per_member": 84.0, + "min_members_per_feature": 14.0, + "mean_members_per_feature": 22.063636779785156, + "max_members_per_feature": 31.0, + "mean_pairwise_jaccard": 0.5278457608823273, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 428, + "metrics": { + "train": { + "0": { + "precision": 0.7830881310769057, + "recall": 0.9845092006319865, + "f1-score": 0.8723225093416136, + "support": 84179.0 + }, + "1": { + "precision": 0.6284900284900284, + "recall": 0.08767188617756935, + "f1-score": 0.15387834821428573, + "support": 25162.0 + }, + "accuracy": 0.7781253143834427, + "macro avg": { + "precision": 0.705789079783467, + "recall": 0.5360905434047779, + "f1-score": 0.5131004287779497, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7475113807518583, + "recall": 0.7781253143834427, + "f1-score": 0.7069911882243216, + "support": 109341.0 + }, + "cross-entropy": 0.4934543091460455, + "roc-auc": 0.6961052398986505, + "score": 0.6961052398986505 + }, + "val": { + "0": { + "precision": 0.6825136099117702, + "recall": 0.9371697383683464, + "f1-score": 0.7898224080812469, + "support": 15518.0 + }, + "1": { + "precision": 0.6698272942770064, + "recall": 0.22623813336383392, + "f1-score": 0.3382352941176471, + "support": 8743.0 + }, + "accuracy": 0.6809694571534561, + "macro avg": { + "precision": 0.6761704520943883, + "recall": 0.5817039358660901, + "f1-score": 0.564028851099447, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6779418091782992, + "recall": 0.6809694571534561, + "f1-score": 0.627082779154832, + "support": 24261.0 + }, + "cross-entropy": 0.6467973900252932, + "roc-auc": 0.6291199291618959, + "score": 0.6291199291618959 + }, + "test": { + "0": { + "precision": 0.661515493867011, + "recall": 0.9688571090887602, + "f1-score": 0.7862178103869947, + "support": 16922.0 + }, + "1": { + "precision": 0.6846199880311191, + "recall": 0.12000419595090737, + "f1-score": 0.20421278114958943, + "support": 9533.0 + }, + "accuracy": 0.662974862974863, + "macro avg": { + "precision": 0.6730677409490651, + "recall": 0.5444306525198338, + "f1-score": 0.49521529576829204, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6698411465930153, + "recall": 0.662974862974863, + "f1-score": 0.5764936015901629, + "support": 26455.0 + }, + "cross-entropy": 0.6863610680565896, + "roc-auc": 0.5961977102213372, + "score": 0.5961977102213372 + } + }, + "time": "0:01:10.628940", + "diversity": { + "train": { + "mean_centered_corr": -0.015634549781680107, + "mean_pairwise_disagreement": 0.03130809179096706, + "member_std": 0.0355970673263073 + }, + "val": { + "mean_centered_corr": -0.0187542624771595, + "mean_pairwise_disagreement": 0.12205770312369114, + "member_std": 0.07598911970853806 + }, + "test": { + "mean_centered_corr": -0.018549803644418716, + "mean_pairwise_disagreement": 0.08149855811146134, + "member_std": 0.0770004391670227 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.7-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.7-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.7-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.7-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..d21da633b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.7-greedy-heads-evaluation/0/report.json @@ -0,0 +1,255 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.6894886493682861, + "min_features_per_member": 67.0, + "mean_features_per_member": 75.84375, + "max_features_per_member": 84.0, + "min_members_per_feature": 14.0, + "mean_members_per_feature": 22.063636779785156, + "max_members_per_feature": 31.0, + "mean_pairwise_jaccard": 0.5278457608823273, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 428, + "metrics": { + "train": { + "0": { + "precision": 0.7894966160281175, + "recall": 0.9686501383955618, + "f1-score": 0.8699455883921904, + "support": 84179.0 + }, + "1": { + "precision": 0.5645214521452145, + "recall": 0.13595898577219617, + "f1-score": 0.21914034975337904, + "support": 25162.0 + }, + "accuracy": 0.7770278303655537, + "macro avg": { + "precision": 0.677009034086666, + "recall": 0.552304562083879, + "f1-score": 0.5445429690727848, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7377244073084094, + "recall": 0.7770278303655537, + "f1-score": 0.7201796139212255, + "support": 109341.0 + }, + "cross-entropy": 0.5002029863194409, + "roc-auc": 0.6821879786641952, + "score": 0.6821879786641952 + }, + "val": { + "0": { + "precision": 0.6961402811584124, + "recall": 0.9030802938523006, + "f1-score": 0.7862212123762238, + "support": 15518.0 + }, + "1": { + "precision": 0.6358353510895883, + "recall": 0.30035456936978155, + "f1-score": 0.4079857065175173, + "support": 8743.0 + }, + "accuracy": 0.6858744487036808, + "macro avg": { + "precision": 0.6659878161240004, + "recall": 0.6017174316110411, + "f1-score": 0.5971034594468705, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6744080358432264, + "recall": 0.6858744487036808, + "f1-score": 0.6499154942391862, + "support": 24261.0 + }, + "cross-entropy": 0.6154470541736884, + "roc-auc": 0.6547032702847417, + "score": 0.6547032702847417 + }, + "test": { + "0": { + "precision": 0.6733296707940667, + "recall": 0.9415553717054722, + "f1-score": 0.7851669335961562, + "support": 16922.0 + }, + "1": { + "precision": 0.6457736389684814, + "recall": 0.18913248714990036, + "f1-score": 0.2925760649087221, + "support": 9533.0 + }, + "accuracy": 0.6704214704214704, + "macro avg": { + "precision": 0.6595516548812741, + "recall": 0.5653439294276863, + "f1-score": 0.5388714992524392, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6633999164416454, + "recall": 0.6704214704214704, + "f1-score": 0.6076629172968816, + "support": 26455.0 + }, + "cross-entropy": 0.638677137478482, + "roc-auc": 0.6118564029158264, + "score": 0.6118564029158264 + } + }, + "time": "0:01:10.628940", + "diversity": { + "train": { + "mean_centered_corr": -0.015634549781680107, + "mean_pairwise_disagreement": 0.03130809179096706, + "member_std": 0.0355970673263073 + }, + "val": { + "mean_centered_corr": -0.0187542624771595, + "mean_pairwise_disagreement": 0.12205770312369114, + "member_std": 0.07598911970853806 + }, + "test": { + "mean_centered_corr": -0.018549803644418716, + "mean_pairwise_disagreement": 0.08149855811146134, + "member_std": 0.0770004391670227 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 16 + ], + "head_selection_time": "0:00:11.162732" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.8-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.8-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.8-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.8-best-head-evaluation/0/report.json new file mode 100644 index 000000000..0237bf07a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.8-best-head-evaluation/0/report.json @@ -0,0 +1,255 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.7894886136054993, + "min_features_per_member": 82.0, + "mean_features_per_member": 86.84375, + "max_features_per_member": 93.0, + "min_members_per_feature": 18.0, + "mean_members_per_feature": 25.263635635375977, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.6542975594370393, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7923570623683308, + "recall": 0.9606196319747206, + "f1-score": 0.8684128569434153, + "support": 84179.0 + }, + "1": { + "precision": 0.5450178424375515, + "recall": 0.15781734361338526, + "f1-score": 0.24476084812623275, + "support": 25162.0 + }, + "accuracy": 0.7758754721467702, + "macro avg": { + "precision": 0.6686874524029411, + "recall": 0.5592184877940529, + "f1-score": 0.556586852534824, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7354383452183298, + "recall": 0.7758754721467702, + "f1-score": 0.7248954952414192, + "support": 109341.0 + }, + "cross-entropy": 0.5019736660581554, + "roc-auc": 0.679122291624921, + "score": 0.679122291624921 + }, + "val": { + "0": { + "precision": 0.7086274721129493, + "recall": 0.8474030158525583, + "f1-score": 0.7718268525311812, + "support": 15518.0 + }, + "1": { + "precision": 0.5848527349228612, + "recall": 0.38156239277135995, + "f1-score": 0.4618259846334879, + "support": 8743.0 + }, + "accuracy": 0.6795268125798607, + "macro avg": { + "precision": 0.6467401035179052, + "recall": 0.6144827043119592, + "f1-score": 0.6168264185823346, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6640224464646685, + "recall": 0.6795268125798607, + "f1-score": 0.6601110292745335, + "support": 24261.0 + }, + "cross-entropy": 0.6209465653764391, + "roc-auc": 0.652377048657135, + "score": 0.652377048657135 + }, + "test": { + "0": { + "precision": 0.6830873199359773, + "recall": 0.9079305046684789, + "f1-score": 0.7796214543055767, + "support": 16922.0 + }, + "1": { + "precision": 0.6068634872571285, + "recall": 0.25228154830588484, + "f1-score": 0.3564018968583284, + "support": 9533.0 + }, + "accuracy": 0.6716688716688717, + "macro avg": { + "precision": 0.6449754035965529, + "recall": 0.5801060264871819, + "f1-score": 0.5680116755819525, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6556202325450318, + "recall": 0.6716688716688717, + "f1-score": 0.6271152346440906, + "support": 26455.0 + }, + "cross-entropy": 0.6427730061058482, + "roc-auc": 0.6066780069996902, + "score": 0.6066780069996902 + } + }, + "time": "0:01:07.254601", + "diversity": { + "train": { + "mean_centered_corr": -0.014071253128349781, + "mean_pairwise_disagreement": 0.02823395645053607, + "member_std": 0.03310052305459976 + }, + "val": { + "mean_centered_corr": -0.021914731711149216, + "mean_pairwise_disagreement": 0.10051144076980047, + "member_std": 0.07142822444438934 + }, + "test": { + "mean_centered_corr": -0.023642798885703087, + "mean_pairwise_disagreement": 0.0652579547740838, + "member_std": 0.07157552242279053 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 24 + ], + "head_selection_time": "0:00:04.881467" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.8-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.8-evaluation/0.toml new file mode 100644 index 000000000..5c75f01be --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.8-evaluation/0.toml @@ -0,0 +1,46 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.8 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.8-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.8-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.8-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.8-evaluation/0/report.json new file mode 100644 index 000000000..92fd4b88b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.8-evaluation/0/report.json @@ -0,0 +1,251 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.7894886136054993, + "min_features_per_member": 82.0, + "mean_features_per_member": 86.84375, + "max_features_per_member": 93.0, + "min_members_per_feature": 18.0, + "mean_members_per_feature": 25.263635635375977, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.6542975594370393, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7822859783644097, + "recall": 0.9853407619477542, + "f1-score": 0.8721504878869448, + "support": 84179.0 + }, + "1": { + "precision": 0.6274154589371981, + "recall": 0.08258485017089262, + "f1-score": 0.1459577158109152, + "support": 25162.0 + }, + "accuracy": 0.7775948637747963, + "macro avg": { + "precision": 0.704850718650804, + "recall": 0.5339628060593234, + "f1-score": 0.5090541018489301, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7466465383572076, + "recall": 0.7775948637747963, + "f1-score": 0.7050360245934222, + "support": 109341.0 + }, + "cross-entropy": 0.49400328229016993, + "roc-auc": 0.6954429108993698, + "score": 0.6954429108993698 + }, + "val": { + "0": { + "precision": 0.6779777757369823, + "recall": 0.9514757056321691, + "f1-score": 0.7917739167739167, + "support": 15518.0 + }, + "1": { + "precision": 0.6967378171566653, + "recall": 0.19787258378131076, + "f1-score": 0.30821307678603244, + "support": 8743.0 + }, + "accuracy": 0.6798977783273566, + "macro avg": { + "precision": 0.6873577964468238, + "recall": 0.5746741447067399, + "f1-score": 0.5499934967799746, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6847383809112244, + "recall": 0.6798977783273566, + "f1-score": 0.6175118326053305, + "support": 24261.0 + }, + "cross-entropy": 0.6534319461189932, + "roc-auc": 0.6293880905914133, + "score": 0.6293880905914133 + }, + "test": { + "0": { + "precision": 0.6589364254298281, + "recall": 0.9738801560099279, + "f1-score": 0.7860345320995898, + "support": 16922.0 + }, + "1": { + "precision": 0.6941176470588235, + "recall": 0.10521346900241267, + "f1-score": 0.18272909455274186, + "support": 9533.0 + }, + "accuracy": 0.6608580608580609, + "macro avg": { + "precision": 0.6765270362443259, + "recall": 0.5395468125061703, + "f1-score": 0.48438181332616587, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6716138998501348, + "recall": 0.6608580608580609, + "f1-score": 0.5686347688739576, + "support": 26455.0 + }, + "cross-entropy": 0.7009929789782234, + "roc-auc": 0.5962685705138886, + "score": 0.5962685705138886 + } + }, + "time": "0:01:07.254601", + "diversity": { + "train": { + "mean_centered_corr": -0.014071253128349781, + "mean_pairwise_disagreement": 0.02823395645053607, + "member_std": 0.03310052305459976 + }, + "val": { + "mean_centered_corr": -0.021914731711149216, + "mean_pairwise_disagreement": 0.10051144076980047, + "member_std": 0.07142822444438934 + }, + "test": { + "mean_centered_corr": -0.023642798885703087, + "mean_pairwise_disagreement": 0.0652579547740838, + "member_std": 0.07157552242279053 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.8-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.8-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.8-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.8-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..e66cb847a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.8-greedy-heads-evaluation/0/report.json @@ -0,0 +1,256 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.7894886136054993, + "min_features_per_member": 82.0, + "mean_features_per_member": 86.84375, + "max_features_per_member": 93.0, + "min_members_per_feature": 18.0, + "mean_members_per_feature": 25.263635635375977, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.6542975594370393, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7922584047977184, + "recall": 0.9635776143693795, + "f1-score": 0.8695600902653824, + "support": 84179.0 + }, + "1": { + "precision": 0.5594194568185084, + "recall": 0.15471743104681662, + "f1-score": 0.24239594035054948, + "support": 25162.0 + }, + "accuracy": 0.777439386872262, + "macro avg": { + "precision": 0.6758389308081134, + "recall": 0.5591475227080981, + "f1-score": 0.555978015307966, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7386765497840191, + "recall": 0.777439386872262, + "f1-score": 0.7252345002291012, + "support": 109341.0 + }, + "cross-entropy": 0.4993136444932815, + "roc-auc": 0.6856466838728517, + "score": 0.6856466838728517 + }, + "val": { + "0": { + "precision": 0.7056227009984235, + "recall": 0.8653176955793273, + "f1-score": 0.7773532476554359, + "support": 15518.0 + }, + "1": { + "precision": 0.6004588032880902, + "recall": 0.35925883563994054, + "f1-score": 0.44954916273078577, + "support": 8743.0 + }, + "accuracy": 0.6829479411401014, + "macro avg": { + "precision": 0.6530407521432569, + "recall": 0.6122882656096339, + "f1-score": 0.6134512051931108, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6677245122312069, + "recall": 0.6829479411401014, + "f1-score": 0.659221632532555, + "support": 24261.0 + }, + "cross-entropy": 0.6165028482842303, + "roc-auc": 0.6540790454616192, + "score": 0.6540790454616192 + }, + "test": { + "0": { + "precision": 0.6807331442286711, + "recall": 0.9218177520387661, + "f1-score": 0.7831413008007632, + "support": 16922.0 + }, + "1": { + "precision": 0.6262711864406779, + "recall": 0.2325605790412252, + "f1-score": 0.3391723399372753, + "support": 9533.0 + }, + "accuracy": 0.6734454734454735, + "macro avg": { + "precision": 0.6535021653346745, + "recall": 0.5771891655399957, + "f1-score": 0.5611568203690193, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6611078997156135, + "recall": 0.6734454734454735, + "f1-score": 0.6231580800896828, + "support": 26455.0 + }, + "cross-entropy": 0.635872441430285, + "roc-auc": 0.6102977368359448, + "score": 0.6102977368359448 + } + }, + "time": "0:01:07.254601", + "diversity": { + "train": { + "mean_centered_corr": -0.014071253128349781, + "mean_pairwise_disagreement": 0.02823395645053607, + "member_std": 0.03310052305459976 + }, + "val": { + "mean_centered_corr": -0.021914731711149216, + "mean_pairwise_disagreement": 0.10051144076980047, + "member_std": 0.07142822444438934 + }, + "test": { + "mean_centered_corr": -0.023642798885703087, + "mean_pairwise_disagreement": 0.0652579547740838, + "member_std": 0.07157552242279053 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 24, + 11 + ], + "head_selection_time": "0:00:15.939899" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.9-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.9-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.9-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.9-best-head-evaluation/0/report.json new file mode 100644 index 000000000..841a9dab2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.9-best-head-evaluation/0/report.json @@ -0,0 +1,255 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.887499988079071, + "min_features_per_member": 92.0, + "mean_features_per_member": 97.625, + "max_features_per_member": 103.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.399999618530273, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.797659951359852, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7919161090894863, + "recall": 0.9617125411325865, + "f1-score": 0.8685939905690236, + "support": 84179.0 + }, + "1": { + "precision": 0.5468859834106565, + "recall": 0.15459820364041013, + "f1-score": 0.24105344694035633, + "support": 25162.0 + }, + "accuracy": 0.77597607484841, + "macro avg": { + "precision": 0.6694010462500715, + "recall": 0.5581553723864984, + "f1-score": 0.5548237187546899, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7355287701925426, + "recall": 0.77597607484841, + "f1-score": 0.7241817832654089, + "support": 109341.0 + }, + "cross-entropy": 0.5019382114215071, + "roc-auc": 0.6778537614421274, + "score": 0.6778537614421274 + }, + "val": { + "0": { + "precision": 0.7093391589295467, + "recall": 0.8369635262276066, + "f1-score": 0.7678845926451461, + "support": 15518.0 + }, + "1": { + "precision": 0.5748613678373382, + "recall": 0.39128445613633767, + "f1-score": 0.46563223084252076, + "support": 8743.0 + }, + "accuracy": 0.6763529945179506, + "macro avg": { + "precision": 0.6421002633834425, + "recall": 0.6141239911819721, + "f1-score": 0.6167584117438334, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.660877045763594, + "recall": 0.6763529945179506, + "f1-score": 0.6589611187883243, + "support": 24261.0 + }, + "cross-entropy": 0.6229299585114317, + "roc-auc": 0.6501781765294031, + "score": 0.6501781765294031 + }, + "test": { + "0": { + "precision": 0.6828953229398663, + "recall": 0.9059803805696726, + "f1-score": 0.7787767956923702, + "support": 16922.0 + }, + "1": { + "precision": 0.6027465667915106, + "recall": 0.25322563726004405, + "f1-score": 0.3566257940611612, + "support": 9533.0 + }, + "accuracy": 0.6707616707616708, + "macro avg": { + "precision": 0.6428209448656885, + "recall": 0.5796030089148583, + "f1-score": 0.5677012948767657, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6540138981671476, + "recall": 0.6707616707616708, + "f1-score": 0.6266556277260004, + "support": 26455.0 + }, + "cross-entropy": 0.6442264400252198, + "roc-auc": 0.6048261425892079, + "score": 0.6048261425892079 + } + }, + "time": "0:01:07.634554", + "diversity": { + "train": { + "mean_centered_corr": -0.010898238979279995, + "mean_pairwise_disagreement": 0.026737251557793006, + "member_std": 0.030055008828639984 + }, + "val": { + "mean_centered_corr": -0.008359640836715698, + "mean_pairwise_disagreement": 0.09315727751030128, + "member_std": 0.06754488497972488 + }, + "test": { + "mean_centered_corr": -0.005901505704969168, + "mean_pairwise_disagreement": 0.05854524725492467, + "member_std": 0.06570815294981003 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 24 + ], + "head_selection_time": "0:00:04.826316" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.9-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.9-evaluation/0.toml new file mode 100644 index 000000000..54bfca637 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.9-evaluation/0.toml @@ -0,0 +1,46 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.9-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.9-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.9-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.9-evaluation/0/report.json new file mode 100644 index 000000000..301ce004d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.9-evaluation/0/report.json @@ -0,0 +1,251 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.887499988079071, + "min_features_per_member": 92.0, + "mean_features_per_member": 97.625, + "max_features_per_member": 103.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.399999618530273, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.797659951359852, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7843021876334646, + "recall": 0.9816937716057449, + "f1-score": 0.8719664035791161, + "support": 84179.0 + }, + "1": { + "precision": 0.6124245472837022, + "recall": 0.09677291153326445, + "f1-score": 0.16713569908710274, + "support": 25162.0 + }, + "accuracy": 0.77805214878225, + "macro avg": { + "precision": 0.6983633674585834, + "recall": 0.5392333415695046, + "f1-score": 0.5195510513331094, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7447489991087509, + "recall": 0.77805214878225, + "f1-score": 0.7097678670152651, + "support": 109341.0 + }, + "cross-entropy": 0.4927182387508514, + "roc-auc": 0.698241002315497, + "score": 0.698241002315497 + }, + "val": { + "0": { + "precision": 0.6764142335766423, + "recall": 0.9554710658590024, + "f1-score": 0.792082910411881, + "support": 15518.0 + }, + "1": { + "precision": 0.7048269970098249, + "recall": 0.18872240649662586, + "f1-score": 0.2977264525442079, + "support": 8743.0 + }, + "accuracy": 0.6791558468323647, + "macro avg": { + "precision": 0.6906206152932336, + "recall": 0.5720967361778142, + "f1-score": 0.5449046814780444, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6866534154197781, + "recall": 0.6791558468323647, + "f1-score": 0.6139303812029833, + "support": 24261.0 + }, + "cross-entropy": 0.6745751820826075, + "roc-auc": 0.6250090898119413, + "score": 0.6250090898119413 + }, + "test": { + "0": { + "precision": 0.6571337782014617, + "recall": 0.9776622148682189, + "f1-score": 0.7859755807876859, + "support": 16922.0 + }, + "1": { + "precision": 0.7044566067240031, + "recall": 0.09451379418860799, + "f1-score": 0.16666666666666666, + "support": 9533.0 + }, + "accuracy": 0.6594216594216594, + "macro avg": { + "precision": 0.6807951924627325, + "recall": 0.5360880045284134, + "f1-score": 0.4763211237271763, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6741864534728805, + "recall": 0.6594216594216594, + "f1-score": 0.5628090006207732, + "support": 26455.0 + }, + "cross-entropy": 0.7367691279234707, + "roc-auc": 0.5927112610884333, + "score": 0.5927112610884333 + } + }, + "time": "0:01:07.634554", + "diversity": { + "train": { + "mean_centered_corr": -0.010898238979279995, + "mean_pairwise_disagreement": 0.026737251557793006, + "member_std": 0.030055008828639984 + }, + "val": { + "mean_centered_corr": -0.008359640836715698, + "mean_pairwise_disagreement": 0.09315727751030128, + "member_std": 0.06754488497972488 + }, + "test": { + "mean_centered_corr": -0.005901505704969168, + "mean_pairwise_disagreement": 0.05854524725492467, + "member_std": 0.06570815294981003 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.9-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.9-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.9-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.9-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..df76434d4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.9-greedy-heads-evaluation/0/report.json @@ -0,0 +1,256 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.887499988079071, + "min_features_per_member": 92.0, + "mean_features_per_member": 97.625, + "max_features_per_member": 103.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.399999618530273, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.797659951359852, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7904454637331882, + "recall": 0.9662742489219401, + "f1-score": 0.8695605693729521, + "support": 84179.0 + }, + "1": { + "precision": 0.5589560354202268, + "recall": 0.14299340275017883, + "f1-score": 0.22772872559258203, + "support": 25162.0 + }, + "accuracy": 0.776817479262125, + "macro avg": { + "precision": 0.6747007495767074, + "recall": 0.5546338258360595, + "f1-score": 0.5486446474827671, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7371741657277673, + "recall": 0.776817479262125, + "f1-score": 0.7218595893818996, + "support": 109341.0 + }, + "cross-entropy": 0.4989347278683794, + "roc-auc": 0.6834597818089504, + "score": 0.6834597818089504 + }, + "val": { + "0": { + "precision": 0.7030905642970632, + "recall": 0.8840056708338704, + "f1-score": 0.7832367466955951, + "support": 15518.0 + }, + "1": { + "precision": 0.6210526315789474, + "recall": 0.3374127873727554, + "f1-score": 0.43726376639739123, + "support": 8743.0 + }, + "accuracy": 0.6870285643625572, + "macro avg": { + "precision": 0.6620715979380053, + "recall": 0.6107092291033129, + "f1-score": 0.6102502565464931, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6735263399965609, + "recall": 0.6870285643625572, + "f1-score": 0.6585575592446575, + "support": 24261.0 + }, + "cross-entropy": 0.6200265526927166, + "roc-auc": 0.6506721330887921, + "score": 0.6506721330887921 + }, + "test": { + "0": { + "precision": 0.6765875561257216, + "recall": 0.9349958633731238, + "f1-score": 0.7850745528072047, + "support": 16922.0 + }, + "1": { + "precision": 0.6416938110749185, + "recall": 0.2066505821881884, + "f1-score": 0.312623978417837, + "support": 9533.0 + }, + "accuracy": 0.6725382725382726, + "macro avg": { + "precision": 0.6591406836003201, + "recall": 0.5708232227806561, + "f1-score": 0.5488492656125209, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6640136732465189, + "recall": 0.6725382725382726, + "f1-score": 0.6148280464509831, + "support": 26455.0 + }, + "cross-entropy": 0.6485088259427312, + "roc-auc": 0.6064667434006787, + "score": 0.6064667434006787 + } + }, + "time": "0:01:07.634554", + "diversity": { + "train": { + "mean_centered_corr": -0.010898238979279995, + "mean_pairwise_disagreement": 0.026737251557793006, + "member_std": 0.030055008828639984 + }, + "val": { + "mean_centered_corr": -0.008359640836715698, + "mean_pairwise_disagreement": 0.09315727751030128, + "member_std": 0.06754488497972488 + }, + "test": { + "mean_centered_corr": -0.005901505704969168, + "mean_pairwise_disagreement": 0.05854524725492467, + "member_std": 0.06570815294981003 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 24, + 22 + ], + "head_selection_time": "0:00:15.957292" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.95-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.95-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.95-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.95-best-head-evaluation/0/report.json new file mode 100644 index 000000000..9ee26893d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.95-best-head-evaluation/0/report.json @@ -0,0 +1,255 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9383522868156433, + "min_features_per_member": 98.0, + "mean_features_per_member": 103.21875, + "max_features_per_member": 107.0, + "min_members_per_feature": 25.0, + "mean_members_per_feature": 30.027273178100586, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8836170743012027, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7913176750815127, + "recall": 0.9629598831062379, + "f1-score": 0.8687418616739097, + "support": 84179.0 + }, + "1": { + "precision": 0.548312327973345, + "recall": 0.15042524441618313, + "f1-score": 0.2360829564946203, + "support": 25162.0 + }, + "accuracy": 0.77597607484841, + "macro avg": { + "precision": 0.6698150015274289, + "recall": 0.5566925637612106, + "f1-score": 0.552412409084265, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7353962865453213, + "recall": 0.77597607484841, + "f1-score": 0.7231517959883821, + "support": 109341.0 + }, + "cross-entropy": 0.5020020262037247, + "roc-auc": 0.677216119994803, + "score": 0.677216119994803 + }, + "val": { + "0": { + "precision": 0.7094869155334019, + "recall": 0.8438587446835932, + "f1-score": 0.7708609271523179, + "support": 15518.0 + }, + "1": { + "precision": 0.5825292901447278, + "recall": 0.3867093674939952, + "f1-score": 0.4648381109507115, + "support": 8743.0 + }, + "accuracy": 0.6791146284159763, + "macro avg": { + "precision": 0.6460081028390648, + "recall": 0.6152840560887942, + "f1-score": 0.6178495190515148, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6637348641433859, + "recall": 0.6791146284159763, + "f1-score": 0.6605786847859421, + "support": 24261.0 + }, + "cross-entropy": 0.6205532929394407, + "roc-auc": 0.6503189442353507, + "score": 0.6503189442353507 + }, + "test": { + "0": { + "precision": 0.6828174268330595, + "recall": 0.9085805460347477, + "f1-score": 0.779685083293187, + "support": 16922.0 + }, + "1": { + "precision": 0.6071609954291518, + "recall": 0.2508129654883038, + "f1-score": 0.35498478212456386, + "support": 9533.0 + }, + "accuracy": 0.6715554715554716, + "macro avg": { + "precision": 0.6449892111311056, + "recall": 0.5796967557615258, + "f1-score": 0.5673349327088755, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6555548012207573, + "recall": 0.6715554715554716, + "f1-score": 0.6266452809480543, + "support": 26455.0 + }, + "cross-entropy": 0.640466255418233, + "roc-auc": 0.6050413611236272, + "score": 0.6050413611236272 + } + }, + "time": "0:01:07.102406", + "diversity": { + "train": { + "mean_centered_corr": -0.012264390476047993, + "mean_pairwise_disagreement": 0.0252751933799292, + "member_std": 0.028376618400216103 + }, + "val": { + "mean_centered_corr": -0.007350938394665718, + "mean_pairwise_disagreement": 0.09498742505893569, + "member_std": 0.06712553650140762 + }, + "test": { + "mean_centered_corr": -0.0008085499866865575, + "mean_pairwise_disagreement": 0.06076119826119826, + "member_std": 0.06589463353157043 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 24 + ], + "head_selection_time": "0:00:04.832161" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.95-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.95-evaluation/0.toml new file mode 100644 index 000000000..43b32f400 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.95-evaluation/0.toml @@ -0,0 +1,46 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.95 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.95-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.95-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.95-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.95-evaluation/0/report.json new file mode 100644 index 000000000..9bb1a7364 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.95-evaluation/0/report.json @@ -0,0 +1,251 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9383522868156433, + "min_features_per_member": 98.0, + "mean_features_per_member": 103.21875, + "max_features_per_member": 107.0, + "min_members_per_feature": 25.0, + "mean_members_per_feature": 30.027273178100586, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8836170743012027, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7858464296612756, + "recall": 0.9789496192637118, + "f1-score": 0.8718333924027867, + "support": 84179.0 + }, + "1": { + "precision": 0.6041992405628769, + "recall": 0.10750337810984818, + "f1-score": 0.18252977495866932, + "support": 25162.0 + }, + "accuracy": 0.7784088310880639, + "macro avg": { + "precision": 0.6950228351120762, + "recall": 0.54322649868678, + "f1-score": 0.527181583680728, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7440450324535136, + "recall": 0.7784088310880639, + "f1-score": 0.7132080128824888, + "support": 109341.0 + }, + "cross-entropy": 0.49252182617427764, + "roc-auc": 0.698478119380352, + "score": 0.698478119380352 + }, + "val": { + "0": { + "precision": 0.6768198008950589, + "recall": 0.9550844180951154, + "f1-score": 0.7922279238828309, + "support": 15518.0 + }, + "1": { + "precision": 0.7050359712230215, + "recall": 0.19055244195356286, + "f1-score": 0.30001800828381053, + "support": 8743.0 + }, + "accuracy": 0.6795680309962491, + "macro avg": { + "precision": 0.6909278860590402, + "recall": 0.5728184300243391, + "f1-score": 0.5461229660833207, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6869881359668769, + "recall": 0.6795680309962491, + "f1-score": 0.6148489497233884, + "support": 24261.0 + }, + "cross-entropy": 0.6755424825758085, + "roc-auc": 0.6252559501617828, + "score": 0.6252559501617828 + }, + "test": { + "0": { + "precision": 0.6574199189253637, + "recall": 0.9775440255288973, + "f1-score": 0.786142001710864, + "support": 16922.0 + }, + "1": { + "precision": 0.7061098221191029, + "recall": 0.09577257946082031, + "f1-score": 0.168668021429891, + "support": 9533.0 + }, + "accuracy": 0.6597996597996598, + "macro avg": { + "precision": 0.6817648705222332, + "recall": 0.5366583024948588, + "f1-score": 0.4774050115703775, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6749652164927767, + "recall": 0.6597996597996598, + "f1-score": 0.5636366358435982, + "support": 26455.0 + }, + "cross-entropy": 0.7385854201893112, + "roc-auc": 0.5929062493223765, + "score": 0.5929062493223765 + } + }, + "time": "0:01:07.102406", + "diversity": { + "train": { + "mean_centered_corr": -0.012264390476047993, + "mean_pairwise_disagreement": 0.0252751933799292, + "member_std": 0.028376618400216103 + }, + "val": { + "mean_centered_corr": -0.007350938394665718, + "mean_pairwise_disagreement": 0.09498742505893569, + "member_std": 0.06712553650140762 + }, + "test": { + "mean_centered_corr": -0.0008085499866865575, + "mean_pairwise_disagreement": 0.06076119826119826, + "member_std": 0.06589463353157043 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.95-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.95-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.95-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.95-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..9495dd28d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.95-greedy-heads-evaluation/0/report.json @@ -0,0 +1,256 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9383522868156433, + "min_features_per_member": 98.0, + "mean_features_per_member": 103.21875, + "max_features_per_member": 107.0, + "min_members_per_feature": 25.0, + "mean_members_per_feature": 30.027273178100586, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8836170743012027, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.790086671260664, + "recall": 0.9670464130008672, + "f1-score": 0.8696557912954298, + "support": 84179.0 + }, + "1": { + "precision": 0.5602409638554217, + "recall": 0.14044988474684048, + "f1-score": 0.2245948522402288, + "support": 25162.0 + }, + "accuracy": 0.776826624962274, + "macro avg": { + "precision": 0.6751638175580428, + "recall": 0.5537481488738538, + "f1-score": 0.5471253217678294, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7371936330615374, + "recall": 0.776826624962274, + "f1-score": 0.7212117186373512, + "support": 109341.0 + }, + "cross-entropy": 0.4990311598658022, + "roc-auc": 0.6831394595121877, + "score": 0.6831394595121877 + }, + "val": { + "0": { + "precision": 0.7033659511245454, + "recall": 0.8847145250676633, + "f1-score": 0.7836858178497017, + "support": 15518.0 + }, + "1": { + "precision": 0.6227330240404892, + "recall": 0.33775591902093105, + "f1-score": 0.43796811271783465, + "support": 8743.0 + }, + "accuracy": 0.6876056221919954, + "macro avg": { + "precision": 0.6630494875825172, + "recall": 0.6112352220442971, + "f1-score": 0.6108269652837681, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6743080523777542, + "recall": 0.6876056221919954, + "f1-score": 0.6590986245778698, + "support": 24261.0 + }, + "cross-entropy": 0.6190109506021244, + "roc-auc": 0.6505655760961024, + "score": 0.6505655760961024 + }, + "test": { + "0": { + "precision": 0.6759120972903776, + "recall": 0.9360595674270181, + "f1-score": 0.7849939291820502, + "support": 16922.0 + }, + "1": { + "precision": 0.6417218543046358, + "recall": 0.2032938214622889, + "f1-score": 0.30877081175814547, + "support": 9533.0 + }, + "accuracy": 0.672009072009072, + "macro avg": { + "precision": 0.6588169757975066, + "recall": 0.5696766944446535, + "f1-score": 0.5468823704700978, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6635917198047199, + "recall": 0.672009072009072, + "f1-score": 0.6133879953925176, + "support": 26455.0 + }, + "cross-entropy": 0.6457100309018923, + "roc-auc": 0.6064442039882287, + "score": 0.6064442039882287 + } + }, + "time": "0:01:07.102406", + "diversity": { + "train": { + "mean_centered_corr": -0.012264390476047993, + "mean_pairwise_disagreement": 0.0252751933799292, + "member_std": 0.028376618400216103 + }, + "val": { + "mean_centered_corr": -0.007350938394665718, + "mean_pairwise_disagreement": 0.09498742505893569, + "member_std": 0.06712553650140762 + }, + "test": { + "mean_centered_corr": -0.0008085499866865575, + "mean_pairwise_disagreement": 0.06076119826119826, + "member_std": 0.06589463353157043 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 24, + 22 + ], + "head_selection_time": "0:00:15.988939" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_cf_fisd-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_cf_fisd-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_cf_fisd-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_cf_fisd-best-head-evaluation/0/report.json new file mode 100644 index 000000000..eb9855f03 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_cf_fisd-best-head-evaluation/0/report.json @@ -0,0 +1,399 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7799945571081353, + "recall": 0.9873840268950689, + "f1-score": 0.8715214428017196, + "support": 84179.0 + }, + "1": { + "precision": 0.6179856115107913, + "recall": 0.0682775614021143, + "f1-score": 0.12296900722926062, + "support": 25162.0 + }, + "accuracy": 0.7758754721467702, + "macro avg": { + "precision": 0.6989900843094633, + "recall": 0.5278307941485916, + "f1-score": 0.4972452250154901, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7427123931520679, + "recall": 0.7758754721467702, + "f1-score": 0.6992614819098838, + "support": 109341.0 + }, + "cross-entropy": 0.4998011928817743, + "roc-auc": 0.6838112658195706, + "score": 0.6838112658195706 + }, + "val": { + "0": { + "precision": 0.6737931657928042, + "recall": 0.9606263693774971, + "f1-score": 0.7920408054832369, + "support": 15518.0 + }, + "1": { + "precision": 0.71408516612073, + "recall": 0.1745396317053643, + "f1-score": 0.28051470588235294, + "support": 8743.0 + }, + "accuracy": 0.6773422365112732, + "macro avg": { + "precision": 0.6939391659567671, + "recall": 0.5675830005414307, + "f1-score": 0.5362777556827949, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6883132992937752, + "recall": 0.6773422365112732, + "f1-score": 0.6077008075931859, + "support": 24261.0 + }, + "cross-entropy": 0.648962937315794, + "roc-auc": 0.6377017988002615, + "score": 0.6377017988002615 + }, + "test": { + "0": { + "precision": 0.6573357157037919, + "recall": 0.9783122562344877, + "f1-score": 0.7863300638848647, + "support": 16922.0 + }, + "1": { + "precision": 0.7110236220472441, + "recall": 0.09472359173397671, + "f1-score": 0.16717578450430437, + "support": 9533.0 + }, + "accuracy": 0.6599130599130599, + "macro avg": { + "precision": 0.684179668875518, + "recall": 0.5365179239842321, + "f1-score": 0.47675292419458454, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6766820325124152, + "recall": 0.6599130599130599, + "f1-score": 0.5632192060003484, + "support": 26455.0 + }, + "cross-entropy": 0.6873082211438988, + "roc-auc": 0.6003639247256524, + "score": 0.6003639247256524 + } + }, + "time": "0:01:59.726664", + "diversity": { + "train": { + "mean_centered_corr": -0.01748897135257721, + "mean_pairwise_disagreement": 0.024599776048355382, + "member_std": 0.027759695425629616 + }, + "val": { + "mean_centered_corr": -0.004482195246964693, + "mean_pairwise_disagreement": 0.06812864068310882, + "member_std": 0.05873022601008415 + }, + "test": { + "mean_centered_corr": 0.0009574671857990324, + "mean_pairwise_disagreement": 0.04505863578444223, + "member_std": 0.060155753046274185 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 11 + ], + "head_selection_time": "0:00:04.870529" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_cf_fisd-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_cf_fisd-evaluation/0.toml new file mode 100644 index 000000000..dc1b937df --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_cf_fisd-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "ecom-offers" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_cf_fisd-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_cf_fisd-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_cf_fisd-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_cf_fisd-evaluation/0/report.json new file mode 100644 index 000000000..bbf83da26 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_cf_fisd-evaluation/0/report.json @@ -0,0 +1,395 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7814940049542719, + "recall": 0.9856615070266931, + "f1-score": 0.8717835566062516, + "support": 84179.0 + }, + "1": { + "precision": 0.6192429022082019, + "recall": 0.07801446625864399, + "f1-score": 0.138571226881265, + "support": 25162.0 + }, + "accuracy": 0.7767900421616777, + "macro avg": { + "precision": 0.7003684535812369, + "recall": 0.5318379866426686, + "f1-score": 0.5051773917437583, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7441561148005635, + "recall": 0.7767900421616777, + "f1-score": 0.7030537238761676, + "support": 109341.0 + }, + "cross-entropy": 0.49570326153659916, + "roc-auc": 0.6915131788984843, + "score": 0.6915131788984843 + }, + "val": { + "0": { + "precision": 0.6750283511000227, + "recall": 0.9589508957339863, + "f1-score": 0.7923222319836009, + "support": 15518.0 + }, + "1": { + "precision": 0.7125451263537906, + "recall": 0.18060162415646802, + "f1-score": 0.28816497855643763, + "support": 8743.0 + }, + "accuracy": 0.6784551337537612, + "macro avg": { + "precision": 0.6937867387269067, + "recall": 0.5697762599452272, + "f1-score": 0.5402436052700192, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6885483694852375, + "recall": 0.6784551337537612, + "f1-score": 0.6106377644549051, + "support": 24261.0 + }, + "cross-entropy": 0.6742112826823753, + "roc-auc": 0.6227441474841354, + "score": 0.6227441474841354 + }, + "test": { + "0": { + "precision": 0.657082804568528, + "recall": 0.9791395816097388, + "f1-score": 0.7864160615121742, + "support": 16922.0 + }, + "1": { + "precision": 0.7150928167877321, + "recall": 0.0929403125983426, + "f1-score": 0.16450055699962868, + "support": 9533.0 + }, + "accuracy": 0.6597996597996598, + "macro avg": { + "precision": 0.6860878106781301, + "recall": 0.5360399471040407, + "f1-score": 0.4754583092559015, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6779865825494644, + "recall": 0.6597996597996598, + "f1-score": 0.5623102023355311, + "support": 26455.0 + }, + "cross-entropy": 0.734778075293986, + "roc-auc": 0.5911932818714825, + "score": 0.5911932818714825 + } + }, + "time": "0:01:59.726664", + "diversity": { + "train": { + "mean_centered_corr": -0.01748897135257721, + "mean_pairwise_disagreement": 0.024599776048355382, + "member_std": 0.027759695425629616 + }, + "val": { + "mean_centered_corr": -0.004482195246964693, + "mean_pairwise_disagreement": 0.06812864068310882, + "member_std": 0.05873022601008415 + }, + "test": { + "mean_centered_corr": 0.0009574671857990324, + "mean_pairwise_disagreement": 0.04505863578444223, + "member_std": 0.060155753046274185 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_cf_fisd-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_cf_fisd-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_cf_fisd-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_cf_fisd-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..8b014d4b0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_cf_fisd-greedy-heads-evaluation/0/report.json @@ -0,0 +1,401 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7815670375259287, + "recall": 0.9847230306846125, + "f1-score": 0.8714616876665668, + "support": 84179.0 + }, + "1": { + "precision": 0.6080463273392258, + "recall": 0.07928622526031318, + "f1-score": 0.1402805611222445, + "support": 25162.0 + }, + "accuracy": 0.7763601942546712, + "macro avg": { + "precision": 0.6948066824325773, + "recall": 0.5320046279724628, + "f1-score": 0.5058711243944056, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7416357390220023, + "recall": 0.7763601942546712, + "f1-score": 0.7031992837548755, + "support": 109341.0 + }, + "cross-entropy": 0.49883518991355796, + "roc-auc": 0.6843233749530935, + "score": 0.6843233749530935 + }, + "val": { + "0": { + "precision": 0.6819298326922185, + "recall": 0.9481891996391287, + "f1-score": 0.7933144628656154, + "support": 15518.0 + }, + "1": { + "precision": 0.7004470938897168, + "recall": 0.21502916619009493, + "f1-score": 0.3290452437210116, + "support": 8743.0 + }, + "accuracy": 0.6839784015498125, + "macro avg": { + "precision": 0.6911884632909677, + "recall": 0.5816091829146118, + "f1-score": 0.5611798532933134, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.688602946523047, + "recall": 0.6839784015498125, + "f1-score": 0.626004550537959, + "support": 24261.0 + }, + "cross-entropy": 0.6356156215085841, + "roc-auc": 0.6386263098818863, + "score": 0.6386263098818863 + }, + "test": { + "0": { + "precision": 0.6633519282076158, + "recall": 0.9697435291336721, + "f1-score": 0.7878060489678349, + "support": 16922.0 + }, + "1": { + "precision": 0.7018054746651136, + "recall": 0.12640302108465332, + "f1-score": 0.21422222222222223, + "support": 9533.0 + }, + "accuracy": 0.6658476658476659, + "macro avg": { + "precision": 0.6825787014363647, + "recall": 0.5480732751091627, + "f1-score": 0.5010141355950286, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6772085775510037, + "recall": 0.6658476658476659, + "f1-score": 0.5811164016283555, + "support": 26455.0 + }, + "cross-entropy": 0.6724107525253293, + "roc-auc": 0.6021922454924368, + "score": 0.6021922454924368 + } + }, + "time": "0:01:59.726664", + "diversity": { + "train": { + "mean_centered_corr": -0.01748897135257721, + "mean_pairwise_disagreement": 0.024599776048355382, + "member_std": 0.027759695425629616 + }, + "val": { + "mean_centered_corr": -0.004482195246964693, + "mean_pairwise_disagreement": 0.06812864068310882, + "member_std": 0.05873022601008415 + }, + "test": { + "mean_centered_corr": 0.0009574671857990324, + "mean_pairwise_disagreement": 0.04505863578444223, + "member_std": 0.060155753046274185 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 11, + 17, + 14 + ], + "head_selection_time": "0:00:20.678561" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_esam-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_esam-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_esam-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_esam-best-head-evaluation/0/report.json new file mode 100644 index 000000000..7d88db838 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_esam-best-head-evaluation/0/report.json @@ -0,0 +1,238 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_esam-evaluation/_esam_diagnostics/rla_esam-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7795739735912358, + "recall": 0.9882037087634683, + "f1-score": 0.8715778003625201, + "support": 84179.0 + }, + "1": { + "precision": 0.623006833712984, + "recall": 0.06521739130434782, + "f1-score": 0.11807454309972658, + "support": 25162.0 + }, + "accuracy": 0.7758023065455776, + "macro avg": { + "precision": 0.7012904036521099, + "recall": 0.5267105500339081, + "f1-score": 0.4948261717311233, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7435441003175637, + "recall": 0.7758023065455776, + "f1-score": 0.6981785360495321, + "support": 109341.0 + }, + "cross-entropy": 0.5001891736479179, + "roc-auc": 0.6828250183964067, + "score": 0.6828250183964067 + }, + "val": { + "0": { + "precision": 0.6731150793650794, + "recall": 0.9619151952571208, + "f1-score": 0.7920093383562371, + "support": 15518.0 + }, + "1": { + "precision": 0.7165467625899281, + "recall": 0.17087956079149033, + "f1-score": 0.2759512375323236, + "support": 8743.0 + }, + "accuracy": 0.6768476155146119, + "macro avg": { + "precision": 0.6948309209775038, + "recall": 0.5663973780243056, + "f1-score": 0.5339802879442803, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6887666686002656, + "recall": 0.6768476155146119, + "f1-score": 0.6060361313365975, + "support": 24261.0 + }, + "cross-entropy": 0.6456461560255325, + "roc-auc": 0.6394659925462143, + "score": 0.6394659925462143 + }, + "test": { + "0": { + "precision": 0.6568744053282588, + "recall": 0.9791395816097388, + "f1-score": 0.78626678688369, + "support": 16922.0 + }, + "1": { + "precision": 0.7132412672623883, + "recall": 0.09210112241686773, + "f1-score": 0.16313638052768487, + "support": 9533.0 + }, + "accuracy": 0.6594972594972595, + "macro avg": { + "precision": 0.6850578362953236, + "recall": 0.5356203520133033, + "f1-score": 0.47470158370568744, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6771860777840538, + "recall": 0.6594972594972595, + "f1-score": 0.5617231405487138, + "support": 26455.0 + }, + "cross-entropy": 0.6814826293175997, + "roc-auc": 0.601268355843962, + "score": 0.601268355843962 + } + }, + "time": "0:02:55.716058", + "diversity": { + "train": { + "mean_centered_corr": -0.01672052964568138, + "mean_pairwise_disagreement": 0.02467565216955184, + "member_std": 0.02795952372252941 + }, + "val": { + "mean_centered_corr": 0.002118270844221115, + "mean_pairwise_disagreement": 0.06719424577610954, + "member_std": 0.05840356647968292 + }, + "test": { + "mean_centered_corr": 0.008818755857646465, + "mean_pairwise_disagreement": 0.04434455039293749, + "member_std": 0.05988753214478493 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 11 + ], + "head_selection_time": "0:00:04.867127" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_esam-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_esam-evaluation/0.toml new file mode 100644 index 000000000..5b186ba93 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_esam-evaluation/0.toml @@ -0,0 +1,50 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_esam-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_esam-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_esam-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_esam-evaluation/0/report.json new file mode 100644 index 000000000..5f2e1b197 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_esam-evaluation/0/report.json @@ -0,0 +1,234 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_esam-evaluation/_esam_diagnostics/rla_esam-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7813565066752028, + "recall": 0.9858872165266872, + "f1-score": 0.8717862526458431, + "support": 84179.0 + }, + "1": { + "precision": 0.6200831467860569, + "recall": 0.0770606470073921, + "f1-score": 0.13708508607585987, + "support": 25162.0 + }, + "accuracy": 0.7767443136609323, + "macro avg": { + "precision": 0.7007198267306298, + "recall": 0.5314739317670396, + "f1-score": 0.5044356693608515, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7442436187234676, + "recall": 0.7767443136609323, + "f1-score": 0.7027138026661107, + "support": 109341.0 + }, + "cross-entropy": 0.4956990087527342, + "roc-auc": 0.6915444543929165, + "score": 0.6915444543929165 + }, + "val": { + "0": { + "precision": 0.6749523723124377, + "recall": 0.9588864544400052, + "f1-score": 0.7922478969225855, + "support": 15518.0 + }, + "1": { + "precision": 0.7119638826185102, + "recall": 0.1803728697243509, + "f1-score": 0.2878262456652674, + "support": 8743.0 + }, + "accuracy": 0.6783314785045959, + "macro avg": { + "precision": 0.6934581274654739, + "recall": 0.569629662082178, + "f1-score": 0.5400370712939264, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6882903070474441, + "recall": 0.6783314785045959, + "f1-score": 0.6104681476565729, + "support": 24261.0 + }, + "cross-entropy": 0.6752158783074748, + "roc-auc": 0.6225317005394864, + "score": 0.6225317005394864 + }, + "test": { + "0": { + "precision": 0.6569933396764985, + "recall": 0.9793168656187212, + "f1-score": 0.7864091491481991, + "support": 16922.0 + }, + "1": { + "precision": 0.7156783103168156, + "recall": 0.0924158187349208, + "f1-score": 0.16369379412857674, + "support": 9533.0 + }, + "accuracy": 0.6597240597240597, + "macro avg": { + "precision": 0.6863358249966571, + "recall": 0.535866342176821, + "f1-score": 0.47505147163838796, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6781403374128109, + "recall": 0.6597240597240597, + "f1-score": 0.5620150656327177, + "support": 26455.0 + }, + "cross-entropy": 0.7364672294386888, + "roc-auc": 0.5910259812848737, + "score": 0.5910259812848737 + } + }, + "time": "0:02:55.716058", + "diversity": { + "train": { + "mean_centered_corr": -0.01672052964568138, + "mean_pairwise_disagreement": 0.02467565216955184, + "member_std": 0.02795952372252941 + }, + "val": { + "mean_centered_corr": 0.002118270844221115, + "mean_pairwise_disagreement": 0.06719424577610954, + "member_std": 0.05840356647968292 + }, + "test": { + "mean_centered_corr": 0.008818755857646465, + "mean_pairwise_disagreement": 0.04434455039293749, + "member_std": 0.05988753214478493 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_esam-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_esam-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_esam-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_esam-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..1605df310 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_esam-greedy-heads-evaluation/0/report.json @@ -0,0 +1,238 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_esam-evaluation/_esam_diagnostics/rla_esam-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7795739735912358, + "recall": 0.9882037087634683, + "f1-score": 0.8715778003625201, + "support": 84179.0 + }, + "1": { + "precision": 0.623006833712984, + "recall": 0.06521739130434782, + "f1-score": 0.11807454309972658, + "support": 25162.0 + }, + "accuracy": 0.7758023065455776, + "macro avg": { + "precision": 0.7012904036521099, + "recall": 0.5267105500339081, + "f1-score": 0.4948261717311233, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7435441003175637, + "recall": 0.7758023065455776, + "f1-score": 0.6981785360495321, + "support": 109341.0 + }, + "cross-entropy": 0.5001891736479179, + "roc-auc": 0.6828250183964067, + "score": 0.6828250183964067 + }, + "val": { + "0": { + "precision": 0.6731150793650794, + "recall": 0.9619151952571208, + "f1-score": 0.7920093383562371, + "support": 15518.0 + }, + "1": { + "precision": 0.7165467625899281, + "recall": 0.17087956079149033, + "f1-score": 0.2759512375323236, + "support": 8743.0 + }, + "accuracy": 0.6768476155146119, + "macro avg": { + "precision": 0.6948309209775038, + "recall": 0.5663973780243056, + "f1-score": 0.5339802879442803, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6887666686002656, + "recall": 0.6768476155146119, + "f1-score": 0.6060361313365975, + "support": 24261.0 + }, + "cross-entropy": 0.6456461560255325, + "roc-auc": 0.6394659925462143, + "score": 0.6394659925462143 + }, + "test": { + "0": { + "precision": 0.6568744053282588, + "recall": 0.9791395816097388, + "f1-score": 0.78626678688369, + "support": 16922.0 + }, + "1": { + "precision": 0.7132412672623883, + "recall": 0.09210112241686773, + "f1-score": 0.16313638052768487, + "support": 9533.0 + }, + "accuracy": 0.6594972594972595, + "macro avg": { + "precision": 0.6850578362953236, + "recall": 0.5356203520133033, + "f1-score": 0.47470158370568744, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6771860777840538, + "recall": 0.6594972594972595, + "f1-score": 0.5617231405487138, + "support": 26455.0 + }, + "cross-entropy": 0.6814826293175997, + "roc-auc": 0.601268355843962, + "score": 0.601268355843962 + } + }, + "time": "0:02:55.716058", + "diversity": { + "train": { + "mean_centered_corr": -0.01672052964568138, + "mean_pairwise_disagreement": 0.02467565216955184, + "member_std": 0.02795952372252941 + }, + "val": { + "mean_centered_corr": 0.002118270844221115, + "mean_pairwise_disagreement": 0.06719424577610954, + "member_std": 0.05840356647968292 + }, + "test": { + "mean_centered_corr": 0.008818755857646465, + "mean_pairwise_disagreement": 0.04434455039293749, + "member_std": 0.05988753214478493 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 11 + ], + "head_selection_time": "0:00:11.170047" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_mfb-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_mfb-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_mfb-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_mfb-best-head-evaluation/0/report.json new file mode 100644 index 000000000..7d6ee9e87 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_mfb-best-head-evaluation/0/report.json @@ -0,0 +1,264 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.887499988079071, + "min_features_per_member": 92.0, + "mean_features_per_member": 97.625, + "max_features_per_member": 103.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.399999618530273, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.797659951359852, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7920123669380088, + "recall": 0.9616412644483778, + "f1-score": 0.86862281167678, + "support": 84179.0 + }, + "1": { + "precision": 0.547315295107248, + "recall": 0.15515459820364041, + "f1-score": 0.2417711720080508, + "support": 25162.0 + }, + "accuracy": 0.7760492404496027, + "macro avg": { + "precision": 0.6696638310226284, + "recall": 0.5583979313260091, + "f1-score": 0.5551969918424154, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7357016717604853, + "recall": 0.7760492404496027, + "f1-score": 0.724369137781859, + "support": 109341.0 + }, + "cross-entropy": 0.5019376417142178, + "roc-auc": 0.677710781278526, + "score": 0.677710781278526 + }, + "val": { + "0": { + "precision": 0.7096030308021742, + "recall": 0.8328392834128109, + "f1-score": 0.7662980640953425, + "support": 15518.0 + }, + "1": { + "precision": 0.5710978835978836, + "recall": 0.39505890426627016, + "f1-score": 0.46704076803461564, + "support": 8743.0 + }, + "accuracy": 0.6750752236099089, + "macro avg": { + "precision": 0.6403504572000289, + "recall": 0.6139490938395405, + "f1-score": 0.616669416064979, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6596895687846518, + "recall": 0.6750752236099089, + "f1-score": 0.6584539299104807, + "support": 24261.0 + }, + "cross-entropy": 0.6233683660120171, + "roc-auc": 0.6500073735640511, + "score": 0.6500073735640511 + }, + "test": { + "0": { + "precision": 0.6827712348322627, + "recall": 0.9044439191584919, + "f1-score": 0.7781280187096447, + "support": 16922.0 + }, + "1": { + "precision": 0.5996533795493935, + "recall": 0.2540648274415189, + "f1-score": 0.3569112879457707, + "support": 9533.0 + }, + "accuracy": 0.67008127008127, + "macro avg": { + "precision": 0.641212307190828, + "recall": 0.5792543733000054, + "f1-score": 0.5675196533277077, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6528199018361716, + "recall": 0.67008127008127, + "f1-score": 0.6263435131578771, + "support": 26455.0 + }, + "cross-entropy": 0.6442461271803751, + "roc-auc": 0.6048362127969982, + "score": 0.6048362127969982 + } + }, + "time": "0:02:06.129189", + "diversity": { + "train": { + "mean_centered_corr": -0.010998313315212727, + "mean_pairwise_disagreement": 0.026646919329909303, + "member_std": 0.02979286201298237 + }, + "val": { + "mean_centered_corr": -0.007380904629826546, + "mean_pairwise_disagreement": 0.09225953042916349, + "member_std": 0.0662473514676094 + }, + "test": { + "mean_centered_corr": -0.004791198763996363, + "mean_pairwise_disagreement": 0.057417952579242905, + "member_std": 0.06390407681465149 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 24 + ], + "head_selection_time": "0:00:04.829893" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_mfb-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_mfb-evaluation/0.toml new file mode 100644 index 000000000..957ed8e36 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_mfb-evaluation/0.toml @@ -0,0 +1,53 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_mfb-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_mfb-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_mfb-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_mfb-evaluation/0/report.json new file mode 100644 index 000000000..505364548 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_mfb-evaluation/0/report.json @@ -0,0 +1,260 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.887499988079071, + "min_features_per_member": 92.0, + "mean_features_per_member": 97.625, + "max_features_per_member": 103.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.399999618530273, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.797659951359852, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7843980518925693, + "recall": 0.9815155798952233, + "f1-score": 0.8719553379205538, + "support": 84179.0 + }, + "1": { + "precision": 0.6117764471057884, + "recall": 0.0974485335029012, + "f1-score": 0.1681179293794995, + "support": 25162.0 + }, + "accuracy": 0.7780704401825482, + "macro avg": { + "precision": 0.6980872494991788, + "recall": 0.5394820566990622, + "f1-score": 0.5200366336500266, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7446736592160346, + "recall": 0.7780704401825482, + "f1-score": 0.7099853827005539, + "support": 109341.0 + }, + "cross-entropy": 0.4926461998052048, + "roc-auc": 0.6983951997801772, + "score": 0.6983951997801772 + }, + "val": { + "0": { + "precision": 0.6763377583139455, + "recall": 0.9554066245650212, + "f1-score": 0.7920083335559176, + "support": 15518.0 + }, + "1": { + "precision": 0.7042735042735043, + "recall": 0.18849365206450874, + "f1-score": 0.29739240277903095, + "support": 8743.0 + }, + "accuracy": 0.6790321915831994, + "macro avg": { + "precision": 0.6903056312937249, + "recall": 0.5719501383147649, + "f1-score": 0.5447003681674742, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6864050361229568, + "recall": 0.6790321915831994, + "f1-score": 0.6137622974163388, + "support": 24261.0 + }, + "cross-entropy": 0.6773464354872277, + "roc-auc": 0.6244884295114916, + "score": 0.6244884295114916 + }, + "test": { + "0": { + "precision": 0.6572257090649083, + "recall": 0.9777213095378797, + "f1-score": 0.7860604332953249, + "support": 16922.0 + }, + "1": { + "precision": 0.7056986729117877, + "recall": 0.09482849050666108, + "f1-score": 0.1671906787497688, + "support": 9533.0 + }, + "accuracy": 0.6595728595728596, + "macro avg": { + "precision": 0.681462190988348, + "recall": 0.5362749000222704, + "f1-score": 0.47662555602254686, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6746928330245492, + "recall": 0.6595728595728596, + "f1-score": 0.5630521032978657, + "support": 26455.0 + }, + "cross-entropy": 0.7412153243698456, + "roc-auc": 0.5923593741199417, + "score": 0.5923593741199417 + } + }, + "time": "0:02:06.129189", + "diversity": { + "train": { + "mean_centered_corr": -0.010998313315212727, + "mean_pairwise_disagreement": 0.026646919329909303, + "member_std": 0.02979286201298237 + }, + "val": { + "mean_centered_corr": -0.007380904629826546, + "mean_pairwise_disagreement": 0.09225953042916349, + "member_std": 0.0662473514676094 + }, + "test": { + "mean_centered_corr": -0.004791198763996363, + "mean_pairwise_disagreement": 0.057417952579242905, + "member_std": 0.06390407681465149 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_mfb-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_mfb-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_mfb-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_mfb-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..c414f631d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_mfb-greedy-heads-evaluation/0/report.json @@ -0,0 +1,265 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.887499988079071, + "min_features_per_member": 92.0, + "mean_features_per_member": 97.625, + "max_features_per_member": 103.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.399999618530273, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.797659951359852, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7905577480122087, + "recall": 0.9661673338956271, + "f1-score": 0.8695852066482409, + "support": 84179.0 + }, + "1": { + "precision": 0.5593377688380009, + "recall": 0.1436690247198156, + "f1-score": 0.22861660079051382, + "support": 25162.0 + }, + "accuracy": 0.7768906448633175, + "macro avg": { + "precision": 0.6749477584251048, + "recall": 0.5549181793077214, + "f1-score": 0.5491009037193774, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.737348456749266, + "recall": 0.7768906448633175, + "f1-score": 0.7220828785133955, + "support": 109341.0 + }, + "cross-entropy": 0.4989600660711784, + "roc-auc": 0.6833803969604821, + "score": 0.6833803969604821 + }, + "val": { + "0": { + "precision": 0.7033559113300493, + "recall": 0.8832968166000773, + "f1-score": 0.7831228932183054, + "support": 15518.0 + }, + "1": { + "precision": 0.6205740624345275, + "recall": 0.3387853139654581, + "f1-score": 0.4382953536549275, + "support": 8743.0 + }, + "accuracy": 0.6870697827789456, + "macro avg": { + "precision": 0.6619649868822883, + "recall": 0.6110410652827677, + "f1-score": 0.6107091234366164, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6735236000117382, + "recall": 0.6870697827789456, + "f1-score": 0.6588564912397137, + "support": 24261.0 + }, + "cross-entropy": 0.6201426085246959, + "roc-auc": 0.6505360715210358, + "score": 0.6505360715210358 + }, + "test": { + "0": { + "precision": 0.6765422598435296, + "recall": 0.9351731473821061, + "f1-score": 0.7851065413142163, + "support": 16922.0 + }, + "1": { + "precision": 0.6419712793733682, + "recall": 0.20633588587013532, + "f1-score": 0.3122965785504485, + "support": 9533.0 + }, + "accuracy": 0.6725382725382726, + "macro avg": { + "precision": 0.6592567696084488, + "recall": 0.5707545166261208, + "f1-score": 0.5487015599323324, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6640846844580808, + "recall": 0.6725382725382726, + "f1-score": 0.6147305301621846, + "support": 26455.0 + }, + "cross-entropy": 0.648405879749753, + "roc-auc": 0.6064993654188358, + "score": 0.6064993654188358 + } + }, + "time": "0:02:06.129189", + "diversity": { + "train": { + "mean_centered_corr": -0.010998313315212727, + "mean_pairwise_disagreement": 0.026646919329909303, + "member_std": 0.02979286201298237 + }, + "val": { + "mean_centered_corr": -0.007380904629826546, + "mean_pairwise_disagreement": 0.09225953042916349, + "member_std": 0.0662473514676094 + }, + "test": { + "mean_centered_corr": -0.004791198763996363, + "mean_pairwise_disagreement": 0.057417952579242905, + "member_std": 0.06390407681465149 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 24, + 22 + ], + "head_selection_time": "0:00:15.977531" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.0001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.0001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.0001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.0001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..16776bce5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.0001-best-head-evaluation/0/report.json @@ -0,0 +1,228 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1951424, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.779700580278046, + "recall": 0.9880492759476829, + "f1-score": 0.8715968394357931, + "support": 84179.0 + }, + "1": { + "precision": 0.6229385307346327, + "recall": 0.06605198314919322, + "f1-score": 0.11943945382680561, + "support": 25162.0 + }, + "accuracy": 0.7758754721467702, + "macro avg": { + "precision": 0.7013195555063394, + "recall": 0.5270506295484381, + "f1-score": 0.4955181466312994, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7436258535917035, + "recall": 0.7758754721467702, + "f1-score": 0.6985072926354772, + "support": 109341.0 + }, + "cross-entropy": 0.5001462491291722, + "roc-auc": 0.6828783854988578, + "score": 0.6828783854988578 + }, + "val": { + "0": { + "precision": 0.6734214920792526, + "recall": 0.9615285474932337, + "f1-score": 0.7920902455209025, + "support": 15518.0 + }, + "1": { + "precision": 0.716254752851711, + "recall": 0.17236646460025162, + "f1-score": 0.2778648474232507, + "support": 8743.0 + }, + "accuracy": 0.677136144429331, + "macro avg": { + "precision": 0.6948381224654818, + "recall": 0.5669475060467426, + "f1-score": 0.5349775464720765, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.688857426250705, + "recall": 0.677136144429331, + "f1-score": 0.6067774943742981, + "support": 24261.0 + }, + "cross-entropy": 0.6457686779603163, + "roc-auc": 0.6396541901648656, + "score": 0.6396541901648656 + }, + "test": { + "0": { + "precision": 0.6569502280388657, + "recall": 0.9789032029310956, + "f1-score": 0.7862448679307972, + "support": 16922.0 + }, + "1": { + "precision": 0.7120967741935483, + "recall": 0.09262561628028952, + "f1-score": 0.16392833936693585, + "support": 9533.0 + }, + "accuracy": 0.6595350595350595, + "macro avg": { + "precision": 0.684523501116207, + "recall": 0.5357644096056925, + "f1-score": 0.4750866036488665, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.67682216243662, + "recall": 0.6595350595350595, + "f1-score": 0.5619945006354167, + "support": 26455.0 + }, + "cross-entropy": 0.6822343746621596, + "roc-auc": 0.6015418321886687, + "score": 0.6015418321886687 + } + }, + "time": "0:01:18.218817", + "diversity": { + "train": { + "mean_centered_corr": -0.016645358875393867, + "mean_pairwise_disagreement": 0.024673918911862297, + "member_std": 0.027964944019913673 + }, + "val": { + "mean_centered_corr": 0.002000282984226942, + "mean_pairwise_disagreement": 0.06748003233651247, + "member_std": 0.05873498693108559 + }, + "test": { + "mean_centered_corr": 0.008286470547318459, + "mean_pairwise_disagreement": 0.04453758969888002, + "member_std": 0.06023459881544113 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 11 + ], + "head_selection_time": "0:00:04.887253" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.0001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.0001-evaluation/0.toml new file mode 100644 index 000000000..2990735ec --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.0001-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.0001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.0001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.0001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.0001-evaluation/0/report.json new file mode 100644 index 000000000..1b45b9b2d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.0001-evaluation/0/report.json @@ -0,0 +1,224 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1951424, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7813988964010622, + "recall": 0.9857921809477423, + "f1-score": 0.8717754770796893, + "support": 84179.0 + }, + "1": { + "precision": 0.6194718421889914, + "recall": 0.0773785867578094, + "f1-score": 0.1375728669846317, + "support": 25162.0 + }, + "accuracy": 0.7767443136609323, + "macro avg": { + "precision": 0.7004353692950268, + "recall": 0.5315853838527759, + "f1-score": 0.5046741720321605, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.744135577626914, + "recall": 0.7767443136609323, + "f1-score": 0.7028177569636135, + "support": 109341.0 + }, + "cross-entropy": 0.4956556906392919, + "roc-auc": 0.6916462356019382, + "score": 0.6916462356019382 + }, + "val": { + "0": { + "precision": 0.6749512449544197, + "recall": 0.9590153370279675, + "f1-score": 0.7922911065562861, + "support": 15518.0 + }, + "1": { + "precision": 0.7124773960216998, + "recall": 0.18025849250829235, + "f1-score": 0.2877225011410315, + "support": 8743.0 + }, + "accuracy": 0.6783726969209843, + "macro avg": { + "precision": 0.6937143204880598, + "recall": 0.5696369147681299, + "f1-score": 0.5400068038486587, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6884746421260627, + "recall": 0.6783726969209843, + "f1-score": 0.6104583990361686, + "support": 24261.0 + }, + "cross-entropy": 0.6750056416383268, + "roc-auc": 0.6227001707049361, + "score": 0.6227001707049361 + }, + "test": { + "0": { + "precision": 0.6569412510901451, + "recall": 0.9793168656187212, + "f1-score": 0.7863718325899213, + "support": 16922.0 + }, + "1": { + "precision": 0.7152156224572823, + "recall": 0.09220602118955208, + "f1-score": 0.163352536703215, + "support": 9533.0 + }, + "accuracy": 0.6596484596484596, + "macro avg": { + "precision": 0.6860784367737137, + "recall": 0.5357614434041367, + "f1-score": 0.47486218464656815, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6779402902979667, + "recall": 0.6596484596484596, + "f1-score": 0.561868224663708, + "support": 26455.0 + }, + "cross-entropy": 0.7361612281771354, + "roc-auc": 0.5911765632808944, + "score": 0.5911765632808944 + } + }, + "time": "0:01:18.218817", + "diversity": { + "train": { + "mean_centered_corr": -0.016645358875393867, + "mean_pairwise_disagreement": 0.024673918911862297, + "member_std": 0.027964944019913673 + }, + "val": { + "mean_centered_corr": 0.002000282984226942, + "mean_pairwise_disagreement": 0.06748003233651247, + "member_std": 0.05873498693108559 + }, + "test": { + "mean_centered_corr": 0.008286470547318459, + "mean_pairwise_disagreement": 0.04453758969888002, + "member_std": 0.06023459881544113 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.0001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.0001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.0001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.0001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..ec841c0fe --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.0001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,228 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1951424, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.779700580278046, + "recall": 0.9880492759476829, + "f1-score": 0.8715968394357931, + "support": 84179.0 + }, + "1": { + "precision": 0.6229385307346327, + "recall": 0.06605198314919322, + "f1-score": 0.11943945382680561, + "support": 25162.0 + }, + "accuracy": 0.7758754721467702, + "macro avg": { + "precision": 0.7013195555063394, + "recall": 0.5270506295484381, + "f1-score": 0.4955181466312994, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7436258535917035, + "recall": 0.7758754721467702, + "f1-score": 0.6985072926354772, + "support": 109341.0 + }, + "cross-entropy": 0.5001462491291722, + "roc-auc": 0.6828783854988578, + "score": 0.6828783854988578 + }, + "val": { + "0": { + "precision": 0.6734214920792526, + "recall": 0.9615285474932337, + "f1-score": 0.7920902455209025, + "support": 15518.0 + }, + "1": { + "precision": 0.716254752851711, + "recall": 0.17236646460025162, + "f1-score": 0.2778648474232507, + "support": 8743.0 + }, + "accuracy": 0.677136144429331, + "macro avg": { + "precision": 0.6948381224654818, + "recall": 0.5669475060467426, + "f1-score": 0.5349775464720765, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.688857426250705, + "recall": 0.677136144429331, + "f1-score": 0.6067774943742981, + "support": 24261.0 + }, + "cross-entropy": 0.6457686779603163, + "roc-auc": 0.6396541901648656, + "score": 0.6396541901648656 + }, + "test": { + "0": { + "precision": 0.6569502280388657, + "recall": 0.9789032029310956, + "f1-score": 0.7862448679307972, + "support": 16922.0 + }, + "1": { + "precision": 0.7120967741935483, + "recall": 0.09262561628028952, + "f1-score": 0.16392833936693585, + "support": 9533.0 + }, + "accuracy": 0.6595350595350595, + "macro avg": { + "precision": 0.684523501116207, + "recall": 0.5357644096056925, + "f1-score": 0.4750866036488665, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.67682216243662, + "recall": 0.6595350595350595, + "f1-score": 0.5619945006354167, + "support": 26455.0 + }, + "cross-entropy": 0.6822343746621596, + "roc-auc": 0.6015418321886687, + "score": 0.6015418321886687 + } + }, + "time": "0:01:18.218817", + "diversity": { + "train": { + "mean_centered_corr": -0.016645358875393867, + "mean_pairwise_disagreement": 0.024673918911862297, + "member_std": 0.027964944019913673 + }, + "val": { + "mean_centered_corr": 0.002000282984226942, + "mean_pairwise_disagreement": 0.06748003233651247, + "member_std": 0.05873498693108559 + }, + "test": { + "mean_centered_corr": 0.008286470547318459, + "mean_pairwise_disagreement": 0.04453758969888002, + "member_std": 0.06023459881544113 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 11 + ], + "head_selection_time": "0:00:11.224192" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..090a927fe --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.001-best-head-evaluation/0/report.json @@ -0,0 +1,228 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1951424, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7796022418414591, + "recall": 0.9881561909739959, + "f1-score": 0.871576984131142, + "support": 84179.0 + }, + "1": { + "precision": 0.6227771471812334, + "recall": 0.06541610364835863, + "f1-score": 0.11839597194749146, + "support": 25162.0 + }, + "accuracy": 0.7758114522457267, + "macro avg": { + "precision": 0.7011896945113463, + "recall": 0.5267861473111772, + "f1-score": 0.4949864780393167, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7435130069539001, + "recall": 0.7758114522457267, + "f1-score": 0.6982518761792758, + "support": 109341.0 + }, + "cross-entropy": 0.5001282298547604, + "roc-auc": 0.6829040213009548, + "score": 0.6829040213009548 + }, + "val": { + "0": { + "precision": 0.6732235506429055, + "recall": 0.9615929887872149, + "f1-score": 0.7919751612132792, + "support": 15518.0 + }, + "1": { + "precision": 0.7156488549618321, + "recall": 0.1715658240878417, + "f1-score": 0.2767783005812344, + "support": 8743.0 + }, + "accuracy": 0.6768888339310004, + "macro avg": { + "precision": 0.6944362028023687, + "recall": 0.5665794064375282, + "f1-score": 0.5343767308972568, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6885124684806028, + "recall": 0.6768888339310004, + "f1-score": 0.6063123215732822, + "support": 24261.0 + }, + "cross-entropy": 0.6459156883389179, + "roc-auc": 0.6395828610304147, + "score": 0.6395828610304147 + }, + "test": { + "0": { + "precision": 0.6568199841395718, + "recall": 0.9789032029310956, + "f1-score": 0.7861515827440558, + "support": 16922.0 + }, + "1": { + "precision": 0.7109311740890688, + "recall": 0.09210112241686773, + "f1-score": 0.16307578008915305, + "support": 9533.0 + }, + "accuracy": 0.6593460593460594, + "macro avg": { + "precision": 0.6838755791143203, + "recall": 0.5355021626739817, + "f1-score": 0.4746136814166044, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6763188302476253, + "recall": 0.6593460593460594, + "f1-score": 0.5616276127304785, + "support": 26455.0 + }, + "cross-entropy": 0.6823614555900946, + "roc-auc": 0.601475038412775, + "score": 0.601475038412775 + } + }, + "time": "0:01:18.777407", + "diversity": { + "train": { + "mean_centered_corr": -0.01685778796672821, + "mean_pairwise_disagreement": 0.024678897417991834, + "member_std": 0.027964096516370773 + }, + "val": { + "mean_centered_corr": 0.0022963290102779865, + "mean_pairwise_disagreement": 0.06754152755451136, + "member_std": 0.05871695652604103 + }, + "test": { + "mean_centered_corr": 0.00823600497096777, + "mean_pairwise_disagreement": 0.04456837843934618, + "member_std": 0.060229621827602386 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 11 + ], + "head_selection_time": "0:00:04.878729" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.001-evaluation/0.toml new file mode 100644 index 000000000..3684da014 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.001-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.001-evaluation/0/report.json new file mode 100644 index 000000000..5037b8574 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.001-evaluation/0/report.json @@ -0,0 +1,224 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1951424, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7813841807909605, + "recall": 0.9857921809477423, + "f1-score": 0.8717663187641495, + "support": 84179.0 + }, + "1": { + "precision": 0.6192295447309774, + "recall": 0.07729910182020507, + "f1-score": 0.137441260643748, + "support": 25162.0 + }, + "accuracy": 0.7767260222606341, + "macro avg": { + "precision": 0.700306862760969, + "recall": 0.5315456413839738, + "f1-score": 0.5046037897039487, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7440684899472578, + "recall": 0.7767260222606341, + "f1-score": 0.702780420405569, + "support": 109341.0 + }, + "cross-entropy": 0.495653101339186, + "roc-auc": 0.6916612924072583, + "score": 0.6916612924072583 + }, + "val": { + "0": { + "precision": 0.67487416678003, + "recall": 0.9590797783219487, + "f1-score": 0.7922599877565143, + "support": 15518.0 + }, + "1": { + "precision": 0.7124094202898551, + "recall": 0.17991536086011667, + "f1-score": 0.28727970048397405, + "support": 8743.0 + }, + "accuracy": 0.6782902600882074, + "macro avg": { + "precision": 0.6936417935349426, + "recall": 0.5694975695910327, + "f1-score": 0.5397698441202442, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6884008442226911, + "recall": 0.6782902600882074, + "f1-score": 0.6102789213691511, + "support": 24261.0 + }, + "cross-entropy": 0.6751155148637754, + "roc-auc": 0.6226233688882503, + "score": 0.6226233688882503 + }, + "test": { + "0": { + "precision": 0.656953694893752, + "recall": 0.9792577709490604, + "f1-score": 0.7863616950600294, + "support": 16922.0 + }, + "1": { + "precision": 0.7148659626320065, + "recall": 0.09231091996223645, + "f1-score": 0.1635079895949461, + "support": 9533.0 + }, + "accuracy": 0.6596484596484596, + "macro avg": { + "precision": 0.6859098287628793, + "recall": 0.5357843454556485, + "f1-score": 0.4749348423274877, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6778222508698919, + "recall": 0.6596484596484596, + "f1-score": 0.561917757271383, + "support": 26455.0 + }, + "cross-entropy": 0.7362984524384457, + "roc-auc": 0.5911217830862241, + "score": 0.5911217830862241 + } + }, + "time": "0:01:18.777407", + "diversity": { + "train": { + "mean_centered_corr": -0.01685778796672821, + "mean_pairwise_disagreement": 0.024678897417991834, + "member_std": 0.027964096516370773 + }, + "val": { + "mean_centered_corr": 0.0022963290102779865, + "mean_pairwise_disagreement": 0.06754152755451136, + "member_std": 0.05871695652604103 + }, + "test": { + "mean_centered_corr": 0.00823600497096777, + "mean_pairwise_disagreement": 0.04456837843934618, + "member_std": 0.060229621827602386 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..095996c3d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,228 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1951424, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7796022418414591, + "recall": 0.9881561909739959, + "f1-score": 0.871576984131142, + "support": 84179.0 + }, + "1": { + "precision": 0.6227771471812334, + "recall": 0.06541610364835863, + "f1-score": 0.11839597194749146, + "support": 25162.0 + }, + "accuracy": 0.7758114522457267, + "macro avg": { + "precision": 0.7011896945113463, + "recall": 0.5267861473111772, + "f1-score": 0.4949864780393167, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7435130069539001, + "recall": 0.7758114522457267, + "f1-score": 0.6982518761792758, + "support": 109341.0 + }, + "cross-entropy": 0.5001282298547604, + "roc-auc": 0.6829040213009548, + "score": 0.6829040213009548 + }, + "val": { + "0": { + "precision": 0.6732235506429055, + "recall": 0.9615929887872149, + "f1-score": 0.7919751612132792, + "support": 15518.0 + }, + "1": { + "precision": 0.7156488549618321, + "recall": 0.1715658240878417, + "f1-score": 0.2767783005812344, + "support": 8743.0 + }, + "accuracy": 0.6768888339310004, + "macro avg": { + "precision": 0.6944362028023687, + "recall": 0.5665794064375282, + "f1-score": 0.5343767308972568, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6885124684806028, + "recall": 0.6768888339310004, + "f1-score": 0.6063123215732822, + "support": 24261.0 + }, + "cross-entropy": 0.6459156883389179, + "roc-auc": 0.6395828610304147, + "score": 0.6395828610304147 + }, + "test": { + "0": { + "precision": 0.6568199841395718, + "recall": 0.9789032029310956, + "f1-score": 0.7861515827440558, + "support": 16922.0 + }, + "1": { + "precision": 0.7109311740890688, + "recall": 0.09210112241686773, + "f1-score": 0.16307578008915305, + "support": 9533.0 + }, + "accuracy": 0.6593460593460594, + "macro avg": { + "precision": 0.6838755791143203, + "recall": 0.5355021626739817, + "f1-score": 0.4746136814166044, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6763188302476253, + "recall": 0.6593460593460594, + "f1-score": 0.5616276127304785, + "support": 26455.0 + }, + "cross-entropy": 0.6823614555900946, + "roc-auc": 0.601475038412775, + "score": 0.601475038412775 + } + }, + "time": "0:01:18.777407", + "diversity": { + "train": { + "mean_centered_corr": -0.01685778796672821, + "mean_pairwise_disagreement": 0.024678897417991834, + "member_std": 0.027964096516370773 + }, + "val": { + "mean_centered_corr": 0.0022963290102779865, + "mean_pairwise_disagreement": 0.06754152755451136, + "member_std": 0.05871695652604103 + }, + "test": { + "mean_centered_corr": 0.00823600497096777, + "mean_pairwise_disagreement": 0.04456837843934618, + "member_std": 0.060229621827602386 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 11 + ], + "head_selection_time": "0:00:11.169700" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise1e-05-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise1e-05-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise1e-05-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise1e-05-best-head-evaluation/0/report.json new file mode 100644 index 000000000..e1fe7d509 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise1e-05-best-head-evaluation/0/report.json @@ -0,0 +1,228 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1951424, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.779683735927936, + "recall": 0.9881205526318916, + "f1-score": 0.8716140457503327, + "support": 84179.0 + }, + "1": { + "precision": 0.6237772761474794, + "recall": 0.06589301327398459, + "f1-score": 0.11919482386772107, + "support": 25162.0 + }, + "accuracy": 0.7758937635470683, + "macro avg": { + "precision": 0.7017305060377077, + "recall": 0.5270067829529381, + "f1-score": 0.4954044348090269, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7438059010718815, + "recall": 0.7758937635470683, + "f1-score": 0.6984642441113292, + "support": 109341.0 + }, + "cross-entropy": 0.5001730088224092, + "roc-auc": 0.682790915855999, + "score": 0.682790915855999 + }, + "val": { + "0": { + "precision": 0.6730699855699855, + "recall": 0.9618507539631396, + "f1-score": 0.791956279513981, + "support": 15518.0 + }, + "1": { + "precision": 0.7160671462829736, + "recall": 0.17076518357543177, + "f1-score": 0.27576653121536754, + "support": 8743.0 + }, + "accuracy": 0.6767651786818351, + "macro avg": { + "precision": 0.6945685659264795, + "recall": 0.5663079687692857, + "f1-score": 0.5338614053646742, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6885649847915204, + "recall": 0.6767651786818351, + "f1-score": 0.6059356303496934, + "support": 24261.0 + }, + "cross-entropy": 0.6459983643302272, + "roc-auc": 0.6395464280764918, + "score": 0.6395464280764918 + }, + "test": { + "0": { + "precision": 0.6567702552719201, + "recall": 0.9791395816097388, + "f1-score": 0.7861921708185053, + "support": 16922.0 + }, + "1": { + "precision": 0.7123064384678076, + "recall": 0.09168152732613029, + "f1-score": 0.162453531598513, + "support": 9533.0 + }, + "accuracy": 0.6593460593460594, + "macro avg": { + "precision": 0.6845383468698638, + "recall": 0.5354105544679345, + "f1-score": 0.47432285120850914, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6767825945048211, + "recall": 0.6593460593460594, + "f1-score": 0.5614293491332214, + "support": 26455.0 + }, + "cross-entropy": 0.6822269705952246, + "roc-auc": 0.6014523595237629, + "score": 0.6014523595237629 + } + }, + "time": "0:01:18.936665", + "diversity": { + "train": { + "mean_centered_corr": -0.016698535531759262, + "mean_pairwise_disagreement": 0.024727871167177205, + "member_std": 0.027984458953142166 + }, + "val": { + "mean_centered_corr": 0.0022652181796729565, + "mean_pairwise_disagreement": 0.06734623868654192, + "member_std": 0.05871646851301193 + }, + "test": { + "mean_centered_corr": 0.008645827881991863, + "mean_pairwise_disagreement": 0.044527987269922756, + "member_std": 0.06025338172912598 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 11 + ], + "head_selection_time": "0:00:04.877353" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise1e-05-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise1e-05-evaluation/0.toml new file mode 100644 index 000000000..b028a7163 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise1e-05-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise1e-05-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise1e-05-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise1e-05-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise1e-05-evaluation/0/report.json new file mode 100644 index 000000000..5c9565e1a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise1e-05-evaluation/0/report.json @@ -0,0 +1,224 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1951424, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7813747645951036, + "recall": 0.9857803015003742, + "f1-score": 0.8717558134037893, + "support": 84179.0 + }, + "1": { + "precision": 0.6189111747851003, + "recall": 0.0772593593514029, + "f1-score": 0.13737059675652757, + "support": 25162.0 + }, + "accuracy": 0.7767077308603361, + "macro avg": { + "precision": 0.7001429696901019, + "recall": 0.5315198304258886, + "f1-score": 0.5045632050801584, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7439879760455265, + "recall": 0.7767077308603361, + "f1-score": 0.7027560711179276, + "support": 109341.0 + }, + "cross-entropy": 0.4956555313079169, + "roc-auc": 0.6916478507195538, + "score": 0.6916478507195538 + }, + "val": { + "0": { + "precision": 0.6748129675810474, + "recall": 0.9590797783219487, + "f1-score": 0.7922178159848827, + "support": 15518.0 + }, + "1": { + "precision": 0.7121486854034451, + "recall": 0.17968660642799955, + "f1-score": 0.28696684628733216, + "support": 8743.0 + }, + "accuracy": 0.6782078232554305, + "macro avg": { + "precision": 0.6934808264922463, + "recall": 0.5693831923749741, + "f1-score": 0.5395923311361075, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6882677378263473, + "recall": 0.6782078232554305, + "f1-score": 0.6101392030643237, + "support": 24261.0 + }, + "cross-entropy": 0.6750260020751795, + "roc-auc": 0.6226529287429354, + "score": 0.6226529287429354 + }, + "test": { + "0": { + "precision": 0.6569412510901451, + "recall": 0.9793168656187212, + "f1-score": 0.7863718325899213, + "support": 16922.0 + }, + "1": { + "precision": 0.7152156224572823, + "recall": 0.09220602118955208, + "f1-score": 0.163352536703215, + "support": 9533.0 + }, + "accuracy": 0.6596484596484596, + "macro avg": { + "precision": 0.6860784367737137, + "recall": 0.5357614434041367, + "f1-score": 0.47486218464656815, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6779402902979667, + "recall": 0.6596484596484596, + "f1-score": 0.561868224663708, + "support": 26455.0 + }, + "cross-entropy": 0.7361401555726955, + "roc-auc": 0.5911485904814773, + "score": 0.5911485904814773 + } + }, + "time": "0:01:18.936665", + "diversity": { + "train": { + "mean_centered_corr": -0.016698535531759262, + "mean_pairwise_disagreement": 0.024727871167177205, + "member_std": 0.027984458953142166 + }, + "val": { + "mean_centered_corr": 0.0022652181796729565, + "mean_pairwise_disagreement": 0.06734623868654192, + "member_std": 0.05871646851301193 + }, + "test": { + "mean_centered_corr": 0.008645827881991863, + "mean_pairwise_disagreement": 0.044527987269922756, + "member_std": 0.06025338172912598 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise1e-05-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise1e-05-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise1e-05-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise1e-05-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..ae3f4b8c0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise1e-05-greedy-heads-evaluation/0/report.json @@ -0,0 +1,228 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1951424, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.779683735927936, + "recall": 0.9881205526318916, + "f1-score": 0.8716140457503327, + "support": 84179.0 + }, + "1": { + "precision": 0.6237772761474794, + "recall": 0.06589301327398459, + "f1-score": 0.11919482386772107, + "support": 25162.0 + }, + "accuracy": 0.7758937635470683, + "macro avg": { + "precision": 0.7017305060377077, + "recall": 0.5270067829529381, + "f1-score": 0.4954044348090269, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7438059010718815, + "recall": 0.7758937635470683, + "f1-score": 0.6984642441113292, + "support": 109341.0 + }, + "cross-entropy": 0.5001730088224092, + "roc-auc": 0.682790915855999, + "score": 0.682790915855999 + }, + "val": { + "0": { + "precision": 0.6730699855699855, + "recall": 0.9618507539631396, + "f1-score": 0.791956279513981, + "support": 15518.0 + }, + "1": { + "precision": 0.7160671462829736, + "recall": 0.17076518357543177, + "f1-score": 0.27576653121536754, + "support": 8743.0 + }, + "accuracy": 0.6767651786818351, + "macro avg": { + "precision": 0.6945685659264795, + "recall": 0.5663079687692857, + "f1-score": 0.5338614053646742, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6885649847915204, + "recall": 0.6767651786818351, + "f1-score": 0.6059356303496934, + "support": 24261.0 + }, + "cross-entropy": 0.6459983643302272, + "roc-auc": 0.6395464280764918, + "score": 0.6395464280764918 + }, + "test": { + "0": { + "precision": 0.6567702552719201, + "recall": 0.9791395816097388, + "f1-score": 0.7861921708185053, + "support": 16922.0 + }, + "1": { + "precision": 0.7123064384678076, + "recall": 0.09168152732613029, + "f1-score": 0.162453531598513, + "support": 9533.0 + }, + "accuracy": 0.6593460593460594, + "macro avg": { + "precision": 0.6845383468698638, + "recall": 0.5354105544679345, + "f1-score": 0.47432285120850914, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6767825945048211, + "recall": 0.6593460593460594, + "f1-score": 0.5614293491332214, + "support": 26455.0 + }, + "cross-entropy": 0.6822269705952246, + "roc-auc": 0.6014523595237629, + "score": 0.6014523595237629 + } + }, + "time": "0:01:18.936665", + "diversity": { + "train": { + "mean_centered_corr": -0.016698535531759262, + "mean_pairwise_disagreement": 0.024727871167177205, + "member_std": 0.027984458953142166 + }, + "val": { + "mean_centered_corr": 0.0022652181796729565, + "mean_pairwise_disagreement": 0.06734623868654192, + "member_std": 0.05871646851301193 + }, + "test": { + "mean_centered_corr": 0.008645827881991863, + "mean_pairwise_disagreement": 0.044527987269922756, + "member_std": 0.06025338172912598 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 11 + ], + "head_selection_time": "0:00:11.193368" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.0001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.0001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.0001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.0001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..c46f9f519 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.0001-best-head-evaluation/0/report.json @@ -0,0 +1,228 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7796513262723779, + "recall": 0.9881443115266277, + "f1-score": 0.8716030366376939, + "support": 84179.0 + }, + "1": { + "precision": 0.6235382874387023, + "recall": 0.06569430092997378, + "f1-score": 0.11886527882644807, + "support": 25162.0 + }, + "accuracy": 0.7758663264466211, + "macro avg": { + "precision": 0.7015948068555401, + "recall": 0.5269193062283007, + "f1-score": 0.49523415773207097, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7437259525961453, + "recall": 0.7758663264466211, + "f1-score": 0.6983799322025179, + "support": 109341.0 + }, + "cross-entropy": 0.500117320045548, + "roc-auc": 0.682915560586896, + "score": 0.682915560586896 + }, + "val": { + "0": { + "precision": 0.6736205183780367, + "recall": 0.9613352236112901, + "f1-score": 0.7921622769753611, + "support": 15518.0 + }, + "1": { + "precision": 0.7163120567375887, + "recall": 0.17328148232872012, + "f1-score": 0.279056916559219, + "support": 8743.0 + }, + "accuracy": 0.6773422365112732, + "macro avg": { + "precision": 0.6949662875578126, + "recall": 0.5673083529700051, + "f1-score": 0.53560959676729, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6890053796730189, + "recall": 0.6773422365112732, + "f1-score": 0.6072531567363632, + "support": 24261.0 + }, + "cross-entropy": 0.6460301333180669, + "roc-auc": 0.6394530055211662, + "score": 0.6394530055211662 + }, + "test": { + "0": { + "precision": 0.656934306569343, + "recall": 0.9786077295827916, + "f1-score": 0.7861381438404937, + "support": 16922.0 + }, + "1": { + "precision": 0.7097032878909383, + "recall": 0.09283541382565824, + "f1-score": 0.16419294990723562, + "support": 9533.0 + }, + "accuracy": 0.6594216594216594, + "macro avg": { + "precision": 0.6833187972301407, + "recall": 0.535721571704225, + "f1-score": 0.4751655468738647, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6759494908044128, + "recall": 0.6594216594216594, + "f1-score": 0.5620215861475908, + "support": 26455.0 + }, + "cross-entropy": 0.6825101151285496, + "roc-auc": 0.6014026252811646, + "score": 0.6014026252811646 + } + }, + "time": "0:01:50.416492", + "diversity": { + "train": { + "mean_centered_corr": -0.01670769229531288, + "mean_pairwise_disagreement": 0.024668442555119807, + "member_std": 0.02791455015540123 + }, + "val": { + "mean_centered_corr": 0.002592463046312332, + "mean_pairwise_disagreement": 0.06743399402465924, + "member_std": 0.05864376202225685 + }, + "test": { + "mean_centered_corr": 0.009284000843763351, + "mean_pairwise_disagreement": 0.0444435468629017, + "member_std": 0.060093436390161514 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 11 + ], + "head_selection_time": "0:00:04.879612" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.0001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.0001-evaluation/0.toml new file mode 100644 index 000000000..f6aef79c4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.0001-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.0001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.0001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.0001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.0001-evaluation/0/report.json new file mode 100644 index 000000000..0574b3d9b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.0001-evaluation/0/report.json @@ -0,0 +1,224 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7814041960149158, + "recall": 0.9857803015003742, + "f1-score": 0.871774130006566, + "support": 84179.0 + }, + "1": { + "precision": 0.61939586645469, + "recall": 0.07741832922661156, + "f1-score": 0.13763380082665067, + "support": 25162.0 + }, + "accuracy": 0.7767443136609323, + "macro avg": { + "precision": 0.7004000312348029, + "recall": 0.5315993153634929, + "f1-score": 0.5047039654166083, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7441221738238402, + "recall": 0.7767443136609323, + "f1-score": 0.7028307422304798, + "support": 109341.0 + }, + "cross-entropy": 0.4956439792677601, + "roc-auc": 0.6916789260357137, + "score": 0.6916789260357137 + }, + "val": { + "0": { + "precision": 0.6749659863945578, + "recall": 0.9590797783219487, + "f1-score": 0.7923232538330494, + "support": 15518.0 + }, + "1": { + "precision": 0.7127996381727725, + "recall": 0.18025849250829235, + "f1-score": 0.28774876757348916, + "support": 8743.0 + }, + "accuracy": 0.6784139153373727, + "macro avg": { + "precision": 0.6938828122836651, + "recall": 0.5696691354151205, + "f1-score": 0.5400360107032692, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6886001984013561, + "recall": 0.6784139153373727, + "f1-score": 0.6104884270176941, + "support": 24261.0 + }, + "cross-entropy": 0.6753608510436575, + "roc-auc": 0.6226365549199251, + "score": 0.6226365549199251 + }, + "test": { + "0": { + "precision": 0.6569933396764985, + "recall": 0.9793168656187212, + "f1-score": 0.7864091491481991, + "support": 16922.0 + }, + "1": { + "precision": 0.7156783103168156, + "recall": 0.0924158187349208, + "f1-score": 0.16369379412857674, + "support": 9533.0 + }, + "accuracy": 0.6597240597240597, + "macro avg": { + "precision": 0.6863358249966571, + "recall": 0.535866342176821, + "f1-score": 0.47505147163838796, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6781403374128109, + "recall": 0.6597240597240597, + "f1-score": 0.5620150656327177, + "support": 26455.0 + }, + "cross-entropy": 0.7367195601018909, + "roc-auc": 0.5911320299643263, + "score": 0.5911320299643263 + } + }, + "time": "0:01:50.416492", + "diversity": { + "train": { + "mean_centered_corr": -0.01670769229531288, + "mean_pairwise_disagreement": 0.024668442555119807, + "member_std": 0.02791455015540123 + }, + "val": { + "mean_centered_corr": 0.002592463046312332, + "mean_pairwise_disagreement": 0.06743399402465924, + "member_std": 0.05864376202225685 + }, + "test": { + "mean_centered_corr": 0.009284000843763351, + "mean_pairwise_disagreement": 0.0444435468629017, + "member_std": 0.060093436390161514 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.0001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.0001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.0001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.0001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..3d6cf2d76 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.0001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,228 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7796513262723779, + "recall": 0.9881443115266277, + "f1-score": 0.8716030366376939, + "support": 84179.0 + }, + "1": { + "precision": 0.6235382874387023, + "recall": 0.06569430092997378, + "f1-score": 0.11886527882644807, + "support": 25162.0 + }, + "accuracy": 0.7758663264466211, + "macro avg": { + "precision": 0.7015948068555401, + "recall": 0.5269193062283007, + "f1-score": 0.49523415773207097, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7437259525961453, + "recall": 0.7758663264466211, + "f1-score": 0.6983799322025179, + "support": 109341.0 + }, + "cross-entropy": 0.500117320045548, + "roc-auc": 0.682915560586896, + "score": 0.682915560586896 + }, + "val": { + "0": { + "precision": 0.6736205183780367, + "recall": 0.9613352236112901, + "f1-score": 0.7921622769753611, + "support": 15518.0 + }, + "1": { + "precision": 0.7163120567375887, + "recall": 0.17328148232872012, + "f1-score": 0.279056916559219, + "support": 8743.0 + }, + "accuracy": 0.6773422365112732, + "macro avg": { + "precision": 0.6949662875578126, + "recall": 0.5673083529700051, + "f1-score": 0.53560959676729, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6890053796730189, + "recall": 0.6773422365112732, + "f1-score": 0.6072531567363632, + "support": 24261.0 + }, + "cross-entropy": 0.6460301333180669, + "roc-auc": 0.6394530055211662, + "score": 0.6394530055211662 + }, + "test": { + "0": { + "precision": 0.656934306569343, + "recall": 0.9786077295827916, + "f1-score": 0.7861381438404937, + "support": 16922.0 + }, + "1": { + "precision": 0.7097032878909383, + "recall": 0.09283541382565824, + "f1-score": 0.16419294990723562, + "support": 9533.0 + }, + "accuracy": 0.6594216594216594, + "macro avg": { + "precision": 0.6833187972301407, + "recall": 0.535721571704225, + "f1-score": 0.4751655468738647, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6759494908044128, + "recall": 0.6594216594216594, + "f1-score": 0.5620215861475908, + "support": 26455.0 + }, + "cross-entropy": 0.6825101151285496, + "roc-auc": 0.6014026252811646, + "score": 0.6014026252811646 + } + }, + "time": "0:01:50.416492", + "diversity": { + "train": { + "mean_centered_corr": -0.01670769229531288, + "mean_pairwise_disagreement": 0.024668442555119807, + "member_std": 0.02791455015540123 + }, + "val": { + "mean_centered_corr": 0.002592463046312332, + "mean_pairwise_disagreement": 0.06743399402465924, + "member_std": 0.05864376202225685 + }, + "test": { + "mean_centered_corr": 0.009284000843763351, + "mean_pairwise_disagreement": 0.0444435468629017, + "member_std": 0.060093436390161514 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 11 + ], + "head_selection_time": "0:00:11.157879" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..a28e18620 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.001-best-head-evaluation/0/report.json @@ -0,0 +1,228 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.779705661792276, + "recall": 0.9881205526318916, + "f1-score": 0.8716277461372007, + "support": 84179.0 + }, + "1": { + "precision": 0.6242014280345735, + "recall": 0.06601224068039106, + "f1-score": 0.11939762067354347, + "support": 25162.0 + }, + "accuracy": 0.7759212006475156, + "macro avg": { + "precision": 0.7019535449134247, + "recall": 0.5270663966561413, + "f1-score": 0.4955126834053721, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7439203888405806, + "recall": 0.7759212006475156, + "f1-score": 0.6985214601427746, + "support": 109341.0 + }, + "cross-entropy": 0.5001030608040564, + "roc-auc": 0.6829250128727141, + "score": 0.6829250128727141 + }, + "val": { + "0": { + "precision": 0.6734841302090387, + "recall": 0.9612707823173089, + "f1-score": 0.792046088087716, + "support": 15518.0 + }, + "1": { + "precision": 0.7154356060606061, + "recall": 0.17282397346448589, + "f1-score": 0.2783970520497467, + "support": 8743.0 + }, + "accuracy": 0.677136144429331, + "macro avg": { + "precision": 0.6944598681348224, + "recall": 0.5670473778908974, + "f1-score": 0.5352215700687314, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6886022932431368, + "recall": 0.677136144429331, + "f1-score": 0.6069410420434489, + "support": 24261.0 + }, + "cross-entropy": 0.6461835001274551, + "roc-auc": 0.6394467257565004, + "score": 0.6394467257565004 + }, + "test": { + "0": { + "precision": 0.6570771001150748, + "recall": 0.9785486349131308, + "f1-score": 0.7862213042755739, + "support": 16922.0 + }, + "1": { + "precision": 0.7105263157894737, + "recall": 0.0934648064617644, + "f1-score": 0.1651988504681561, + "support": 9533.0 + }, + "accuracy": 0.6596106596106596, + "macro avg": { + "precision": 0.6838017079522742, + "recall": 0.5360067206874476, + "f1-score": 0.475710077371865, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6763374052756889, + "recall": 0.6596106596106596, + "f1-score": 0.5624372539204004, + "support": 26455.0 + }, + "cross-entropy": 0.682694952589047, + "roc-auc": 0.6014832923257777, + "score": 0.6014832923257777 + } + }, + "time": "0:01:49.700702", + "diversity": { + "train": { + "mean_centered_corr": -0.01669180765748024, + "mean_pairwise_disagreement": 0.024674711785060706, + "member_std": 0.027917660772800446 + }, + "val": { + "mean_centered_corr": 0.0025723774451762438, + "mean_pairwise_disagreement": 0.06741305240988126, + "member_std": 0.05856572464108467 + }, + "test": { + "mean_centered_corr": 0.00913835596293211, + "mean_pairwise_disagreement": 0.04448012754464367, + "member_std": 0.060016945004463196 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 11 + ], + "head_selection_time": "0:00:04.877772" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.001-evaluation/0.toml new file mode 100644 index 000000000..224095ef4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.001-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.001-evaluation/0/report.json new file mode 100644 index 000000000..17d82eeb4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.001-evaluation/0/report.json @@ -0,0 +1,224 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7814041960149158, + "recall": 0.9857803015003742, + "f1-score": 0.871774130006566, + "support": 84179.0 + }, + "1": { + "precision": 0.61939586645469, + "recall": 0.07741832922661156, + "f1-score": 0.13763380082665067, + "support": 25162.0 + }, + "accuracy": 0.7767443136609323, + "macro avg": { + "precision": 0.7004000312348029, + "recall": 0.5315993153634929, + "f1-score": 0.5047039654166083, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7441221738238402, + "recall": 0.7767443136609323, + "f1-score": 0.7028307422304798, + "support": 109341.0 + }, + "cross-entropy": 0.49563994809608797, + "roc-auc": 0.6916786795898221, + "score": 0.6916786795898221 + }, + "val": { + "0": { + "precision": 0.6749671157073525, + "recall": 0.9589508957339863, + "f1-score": 0.792280047916944, + "support": 15518.0 + }, + "1": { + "precision": 0.7122854561878952, + "recall": 0.1803728697243509, + "f1-score": 0.28785251437437254, + "support": 8743.0 + }, + "accuracy": 0.6783726969209843, + "macro avg": { + "precision": 0.6936262859476239, + "recall": 0.5696618827291686, + "f1-score": 0.5400662811456582, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6884156236345355, + "recall": 0.6783726969209843, + "f1-score": 0.6104981788364154, + "support": 24261.0 + }, + "cross-entropy": 0.6753967669711587, + "roc-auc": 0.622598916870318, + "score": 0.622598916870318 + }, + "test": { + "0": { + "precision": 0.6570318385472423, + "recall": 0.9792577709490604, + "f1-score": 0.78641767316043, + "support": 16922.0 + }, + "1": { + "precision": 0.7155591572123177, + "recall": 0.09262561628028952, + "f1-score": 0.16401968979288567, + "support": 9533.0 + }, + "accuracy": 0.6597618597618597, + "macro avg": { + "precision": 0.68629549787978, + "recall": 0.5359416936146749, + "f1-score": 0.4752186814766578, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.678122026747362, + "recall": 0.6597618597618597, + "f1-score": 0.5621379538089728, + "support": 26455.0 + }, + "cross-entropy": 0.7367599370342945, + "roc-auc": 0.5911027119909538, + "score": 0.5911027119909538 + } + }, + "time": "0:01:49.700702", + "diversity": { + "train": { + "mean_centered_corr": -0.01669180765748024, + "mean_pairwise_disagreement": 0.024674711785060706, + "member_std": 0.027917660772800446 + }, + "val": { + "mean_centered_corr": 0.0025723774451762438, + "mean_pairwise_disagreement": 0.06741305240988126, + "member_std": 0.05856572464108467 + }, + "test": { + "mean_centered_corr": 0.00913835596293211, + "mean_pairwise_disagreement": 0.04448012754464367, + "member_std": 0.060016945004463196 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..40942574c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,228 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.779705661792276, + "recall": 0.9881205526318916, + "f1-score": 0.8716277461372007, + "support": 84179.0 + }, + "1": { + "precision": 0.6242014280345735, + "recall": 0.06601224068039106, + "f1-score": 0.11939762067354347, + "support": 25162.0 + }, + "accuracy": 0.7759212006475156, + "macro avg": { + "precision": 0.7019535449134247, + "recall": 0.5270663966561413, + "f1-score": 0.4955126834053721, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7439203888405806, + "recall": 0.7759212006475156, + "f1-score": 0.6985214601427746, + "support": 109341.0 + }, + "cross-entropy": 0.5001030608040564, + "roc-auc": 0.6829250128727141, + "score": 0.6829250128727141 + }, + "val": { + "0": { + "precision": 0.6734841302090387, + "recall": 0.9612707823173089, + "f1-score": 0.792046088087716, + "support": 15518.0 + }, + "1": { + "precision": 0.7154356060606061, + "recall": 0.17282397346448589, + "f1-score": 0.2783970520497467, + "support": 8743.0 + }, + "accuracy": 0.677136144429331, + "macro avg": { + "precision": 0.6944598681348224, + "recall": 0.5670473778908974, + "f1-score": 0.5352215700687314, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6886022932431368, + "recall": 0.677136144429331, + "f1-score": 0.6069410420434489, + "support": 24261.0 + }, + "cross-entropy": 0.6461835001274551, + "roc-auc": 0.6394467257565004, + "score": 0.6394467257565004 + }, + "test": { + "0": { + "precision": 0.6570771001150748, + "recall": 0.9785486349131308, + "f1-score": 0.7862213042755739, + "support": 16922.0 + }, + "1": { + "precision": 0.7105263157894737, + "recall": 0.0934648064617644, + "f1-score": 0.1651988504681561, + "support": 9533.0 + }, + "accuracy": 0.6596106596106596, + "macro avg": { + "precision": 0.6838017079522742, + "recall": 0.5360067206874476, + "f1-score": 0.475710077371865, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6763374052756889, + "recall": 0.6596106596106596, + "f1-score": 0.5624372539204004, + "support": 26455.0 + }, + "cross-entropy": 0.682694952589047, + "roc-auc": 0.6014832923257777, + "score": 0.6014832923257777 + } + }, + "time": "0:01:49.700702", + "diversity": { + "train": { + "mean_centered_corr": -0.01669180765748024, + "mean_pairwise_disagreement": 0.024674711785060706, + "member_std": 0.027917660772800446 + }, + "val": { + "mean_centered_corr": 0.0025723774451762438, + "mean_pairwise_disagreement": 0.06741305240988126, + "member_std": 0.05856572464108467 + }, + "test": { + "mean_centered_corr": 0.00913835596293211, + "mean_pairwise_disagreement": 0.04448012754464367, + "member_std": 0.060016945004463196 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 11 + ], + "head_selection_time": "0:00:11.147273" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise1e-05-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise1e-05-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise1e-05-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise1e-05-best-head-evaluation/0/report.json new file mode 100644 index 000000000..11f463d90 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise1e-05-best-head-evaluation/0/report.json @@ -0,0 +1,228 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7797560449657319, + "recall": 0.9879898787108423, + "f1-score": 0.8716083798824134, + "support": 84179.0 + }, + "1": { + "precision": 0.6230425055928411, + "recall": 0.06640966536841268, + "f1-score": 0.12002585835368482, + "support": 25162.0 + }, + "accuracy": 0.7759120549473665, + "macro avg": { + "precision": 0.7013992752792866, + "recall": 0.5271997720396275, + "f1-score": 0.4958171191180491, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7436924816390686, + "recall": 0.7759120549473665, + "f1-score": 0.6986511231653002, + "support": 109341.0 + }, + "cross-entropy": 0.5001252189253765, + "roc-auc": 0.682885703100578, + "score": 0.682885703100578 + }, + "val": { + "0": { + "precision": 0.6736970463372776, + "recall": 0.9612707823173089, + "f1-score": 0.7921933085501859, + "support": 15518.0 + }, + "1": { + "precision": 0.7163756488909863, + "recall": 0.1736246139768958, + "f1-score": 0.2795065365494384, + "support": 8743.0 + }, + "accuracy": 0.6774246733440501, + "macro avg": { + "precision": 0.6950363476141319, + "recall": 0.5674476981471024, + "f1-score": 0.5358499225498121, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6890772459220875, + "recall": 0.6774246733440501, + "f1-score": 0.6074350361128364, + "support": 24261.0 + }, + "cross-entropy": 0.6460050337477455, + "roc-auc": 0.6396100364908871, + "score": 0.6396100364908871 + }, + "test": { + "0": { + "precision": 0.6570385653070941, + "recall": 0.9786077295827916, + "f1-score": 0.7862127902008261, + "support": 16922.0 + }, + "1": { + "precision": 0.7106314948041567, + "recall": 0.09325500891639568, + "f1-score": 0.1648738872403561, + "support": 9533.0 + }, + "accuracy": 0.6595728595728596, + "macro avg": { + "precision": 0.6838350300556254, + "recall": 0.5359313692495936, + "f1-score": 0.4755433387205911, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6763506574218361, + "recall": 0.6595728595728596, + "f1-score": 0.5623147081020864, + "support": 26455.0 + }, + "cross-entropy": 0.6825652834520043, + "roc-auc": 0.6014864444960832, + "score": 0.6014864444960832 + } + }, + "time": "0:01:49.922203", + "diversity": { + "train": { + "mean_centered_corr": -0.016829919070005417, + "mean_pairwise_disagreement": 0.02467144809770912, + "member_std": 0.027938073500990868 + }, + "val": { + "mean_centered_corr": 0.002263085450977087, + "mean_pairwise_disagreement": 0.06737382843299547, + "member_std": 0.05858324468135834 + }, + "test": { + "mean_centered_corr": 0.008585128001868725, + "mean_pairwise_disagreement": 0.044407347229927875, + "member_std": 0.06004700809717178 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 11 + ], + "head_selection_time": "0:00:04.887817" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise1e-05-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise1e-05-evaluation/0.toml new file mode 100644 index 000000000..4e35430e6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise1e-05-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise1e-05-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise1e-05-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise1e-05-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise1e-05-evaluation/0/report.json new file mode 100644 index 000000000..14a93ea2b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise1e-05-evaluation/0/report.json @@ -0,0 +1,224 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7814009548112506, + "recall": 0.9858040603951104, + "f1-score": 0.871781403313408, + "support": 84179.0 + }, + "1": { + "precision": 0.6196690006365373, + "recall": 0.0773785867578094, + "f1-score": 0.13757772752967778, + "support": 25162.0 + }, + "accuracy": 0.7767534593610814, + "macro avg": { + "precision": 0.700534977723894, + "recall": 0.5315913235764599, + "f1-score": 0.5046795654215429, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7441825332590045, + "recall": 0.7767534593610814, + "f1-score": 0.702823437956678, + "support": 109341.0 + }, + "cross-entropy": 0.4956495574180934, + "roc-auc": 0.6916692445363317, + "score": 0.6916692445363317 + }, + "val": { + "0": { + "precision": 0.6749965984851921, + "recall": 0.9590797783219487, + "f1-score": 0.7923443447706764, + "support": 15518.0 + }, + "1": { + "precision": 0.7129294755877035, + "recall": 0.1803728697243509, + "f1-score": 0.2879050661798266, + "support": 8743.0 + }, + "accuracy": 0.6784551337537612, + "macro avg": { + "precision": 0.6939630370364478, + "recall": 0.5697263240231498, + "f1-score": 0.5401247054752515, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6886665685815302, + "recall": 0.6784551337537612, + "f1-score": 0.6105582430963925, + "support": 24261.0 + }, + "cross-entropy": 0.6751780870468839, + "roc-auc": 0.6226645595746754, + "score": 0.6226645595746754 + }, + "test": { + "0": { + "precision": 0.6569797407128415, + "recall": 0.9792577709490604, + "f1-score": 0.7863803535413454, + "support": 16922.0 + }, + "1": { + "precision": 0.7150974025974026, + "recall": 0.0924158187349208, + "f1-score": 0.16367858801672086, + "support": 9533.0 + }, + "accuracy": 0.6596862596862597, + "macro avg": { + "precision": 0.686038571655122, + "recall": 0.5358367948419906, + "f1-score": 0.4750294707790331, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6779223100095916, + "recall": 0.6596862596862597, + "f1-score": 0.5619911669699508, + "support": 26455.0 + }, + "cross-entropy": 0.7364469465377833, + "roc-auc": 0.5911489283247057, + "score": 0.5911489283247057 + } + }, + "time": "0:01:49.922203", + "diversity": { + "train": { + "mean_centered_corr": -0.016829919070005417, + "mean_pairwise_disagreement": 0.02467144809770912, + "member_std": 0.027938073500990868 + }, + "val": { + "mean_centered_corr": 0.002263085450977087, + "mean_pairwise_disagreement": 0.06737382843299547, + "member_std": 0.05858324468135834 + }, + "test": { + "mean_centered_corr": 0.008585128001868725, + "mean_pairwise_disagreement": 0.044407347229927875, + "member_std": 0.06004700809717178 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise1e-05-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise1e-05-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise1e-05-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise1e-05-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..13ed1d5d4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise1e-05-greedy-heads-evaluation/0/report.json @@ -0,0 +1,228 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7797560449657319, + "recall": 0.9879898787108423, + "f1-score": 0.8716083798824134, + "support": 84179.0 + }, + "1": { + "precision": 0.6230425055928411, + "recall": 0.06640966536841268, + "f1-score": 0.12002585835368482, + "support": 25162.0 + }, + "accuracy": 0.7759120549473665, + "macro avg": { + "precision": 0.7013992752792866, + "recall": 0.5271997720396275, + "f1-score": 0.4958171191180491, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7436924816390686, + "recall": 0.7759120549473665, + "f1-score": 0.6986511231653002, + "support": 109341.0 + }, + "cross-entropy": 0.5001252189253765, + "roc-auc": 0.682885703100578, + "score": 0.682885703100578 + }, + "val": { + "0": { + "precision": 0.6736970463372776, + "recall": 0.9612707823173089, + "f1-score": 0.7921933085501859, + "support": 15518.0 + }, + "1": { + "precision": 0.7163756488909863, + "recall": 0.1736246139768958, + "f1-score": 0.2795065365494384, + "support": 8743.0 + }, + "accuracy": 0.6774246733440501, + "macro avg": { + "precision": 0.6950363476141319, + "recall": 0.5674476981471024, + "f1-score": 0.5358499225498121, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6890772459220875, + "recall": 0.6774246733440501, + "f1-score": 0.6074350361128364, + "support": 24261.0 + }, + "cross-entropy": 0.6460050337477455, + "roc-auc": 0.6396100364908871, + "score": 0.6396100364908871 + }, + "test": { + "0": { + "precision": 0.6570385653070941, + "recall": 0.9786077295827916, + "f1-score": 0.7862127902008261, + "support": 16922.0 + }, + "1": { + "precision": 0.7106314948041567, + "recall": 0.09325500891639568, + "f1-score": 0.1648738872403561, + "support": 9533.0 + }, + "accuracy": 0.6595728595728596, + "macro avg": { + "precision": 0.6838350300556254, + "recall": 0.5359313692495936, + "f1-score": 0.4755433387205911, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6763506574218361, + "recall": 0.6595728595728596, + "f1-score": 0.5623147081020864, + "support": 26455.0 + }, + "cross-entropy": 0.6825652834520043, + "roc-auc": 0.6014864444960832, + "score": 0.6014864444960832 + } + }, + "time": "0:01:49.922203", + "diversity": { + "train": { + "mean_centered_corr": -0.016829919070005417, + "mean_pairwise_disagreement": 0.02467144809770912, + "member_std": 0.027938073500990868 + }, + "val": { + "mean_centered_corr": 0.002263085450977087, + "mean_pairwise_disagreement": 0.06737382843299547, + "member_std": 0.05858324468135834 + }, + "test": { + "mean_centered_corr": 0.008585128001868725, + "mean_pairwise_disagreement": 0.044407347229927875, + "member_std": 0.06004700809717178 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 11 + ], + "head_selection_time": "0:00:11.225544" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.0001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.0001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.0001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.0001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..93e34fb24 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.0001-best-head-evaluation/0/report.json @@ -0,0 +1,228 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 2457536, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7797414237631374, + "recall": 0.9879898787108423, + "f1-score": 0.8715992454412073, + "support": 84179.0 + }, + "1": { + "precision": 0.6227611940298508, + "recall": 0.06633018043080836, + "f1-score": 0.11989081244163494, + "support": 25162.0 + }, + "accuracy": 0.7758937635470683, + "macro avg": { + "precision": 0.701251308896494, + "recall": 0.5271600295708253, + "f1-score": 0.4957450289414211, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7436164885554023, + "recall": 0.7758937635470683, + "f1-score": 0.6986130134592862, + "support": 109341.0 + }, + "cross-entropy": 0.5001025680620461, + "roc-auc": 0.6830259884114023, + "score": 0.6830259884114023 + }, + "val": { + "0": { + "precision": 0.6732070365358592, + "recall": 0.9617863126691584, + "f1-score": 0.7920292931437062, + "support": 15518.0 + }, + "1": { + "precision": 0.7164036346245816, + "recall": 0.17133706965572457, + "f1-score": 0.2765368285028614, + "support": 8743.0 + }, + "accuracy": 0.6769300523473888, + "macro avg": { + "precision": 0.6948053355802204, + "recall": 0.5665616911624415, + "f1-score": 0.5342830608232838, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6887739075259132, + "recall": 0.6769300523473888, + "f1-score": 0.6062599259142059, + "support": 24261.0 + }, + "cross-entropy": 0.6465081837078666, + "roc-auc": 0.639398116545268, + "score": 0.639398116545268 + }, + "test": { + "0": { + "precision": 0.6569128900519409, + "recall": 0.979080486940078, + "f1-score": 0.7862753007616924, + "support": 16922.0 + }, + "1": { + "precision": 0.713128038897893, + "recall": 0.09231091996223645, + "f1-score": 0.16346243150366863, + "support": 9533.0 + }, + "accuracy": 0.6595350595350595, + "macro avg": { + "precision": 0.6850204644749169, + "recall": 0.5356957034511572, + "f1-score": 0.47486886613268053, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6771698930361957, + "recall": 0.6595350595350595, + "f1-score": 0.5618460782088011, + "support": 26455.0 + }, + "cross-entropy": 0.6832483314751614, + "roc-auc": 0.6013710911801927, + "score": 0.6013710911801927 + } + }, + "time": "0:02:55.144244", + "diversity": { + "train": { + "mean_centered_corr": -0.016687747091054916, + "mean_pairwise_disagreement": 0.024748319920131486, + "member_std": 0.02787911146879196 + }, + "val": { + "mean_centered_corr": 0.00230225152336061, + "mean_pairwise_disagreement": 0.0672119464266957, + "member_std": 0.058351658284664154 + }, + "test": { + "mean_centered_corr": 0.008990149945020676, + "mean_pairwise_disagreement": 0.0442228434163918, + "member_std": 0.059662897139787674 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 11 + ], + "head_selection_time": "0:00:04.808438" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.0001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.0001-evaluation/0.toml new file mode 100644 index 000000000..3cbee9e39 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.0001-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 8 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.0001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.0001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.0001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.0001-evaluation/0/report.json new file mode 100644 index 000000000..d1e209b0f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.0001-evaluation/0/report.json @@ -0,0 +1,224 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 2457536, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7814086204785806, + "recall": 0.9857209042635336, + "f1-score": 0.8717536560766516, + "support": 84179.0 + }, + "1": { + "precision": 0.6186548223350253, + "recall": 0.07749781416421588, + "f1-score": 0.13774104683195593, + "support": 25162.0 + }, + "accuracy": 0.776716876560485, + "macro avg": { + "precision": 0.700031721406803, + "recall": 0.5316093592138748, + "f1-score": 0.5047473514543037, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.743955047995357, + "recall": 0.776716876560485, + "f1-score": 0.7028396597366232, + "support": 109341.0 + }, + "cross-entropy": 0.4956245931886986, + "roc-auc": 0.6917220323493016, + "score": 0.6917220323493016 + }, + "val": { + "0": { + "precision": 0.6747823721436343, + "recall": 0.9590797783219487, + "f1-score": 0.7921967317826156, + "support": 15518.0 + }, + "1": { + "precision": 0.7120181405895691, + "recall": 0.179572229211941, + "f1-score": 0.28681037632444284, + "support": 8743.0 + }, + "accuracy": 0.6781666048390421, + "macro avg": { + "precision": 0.6934002563666017, + "recall": 0.5693260037669449, + "f1-score": 0.5395035540535292, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6882011233708224, + "recall": 0.6781666048390421, + "f1-score": 0.6100693295415371, + "support": 24261.0 + }, + "cross-entropy": 0.6759411491858018, + "roc-auc": 0.6225693459597093, + "score": 0.6225693459597093 + }, + "test": { + "0": { + "precision": 0.6568891707626446, + "recall": 0.9793168656187212, + "f1-score": 0.7863345195729537, + "support": 16922.0 + }, + "1": { + "precision": 0.7147514262428688, + "recall": 0.09199622364418336, + "f1-score": 0.16301115241635689, + "support": 9533.0 + }, + "accuracy": 0.6595728595728596, + "macro avg": { + "precision": 0.6858202985027567, + "recall": 0.5356565446314523, + "f1-score": 0.4746728359946553, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6777397049336135, + "recall": 0.6595728595728596, + "f1-score": 0.5617213402456493, + "support": 26455.0 + }, + "cross-entropy": 0.7376821008229536, + "roc-auc": 0.5910522493707531, + "score": 0.5910522493707531 + } + }, + "time": "0:02:55.144244", + "diversity": { + "train": { + "mean_centered_corr": -0.016687747091054916, + "mean_pairwise_disagreement": 0.024748319920131486, + "member_std": 0.02787911146879196 + }, + "val": { + "mean_centered_corr": 0.00230225152336061, + "mean_pairwise_disagreement": 0.0672119464266957, + "member_std": 0.058351658284664154 + }, + "test": { + "mean_centered_corr": 0.008990149945020676, + "mean_pairwise_disagreement": 0.0442228434163918, + "member_std": 0.059662897139787674 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.0001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.0001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.0001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.0001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..cdcaf1088 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.0001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,228 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 2457536, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7797414237631374, + "recall": 0.9879898787108423, + "f1-score": 0.8715992454412073, + "support": 84179.0 + }, + "1": { + "precision": 0.6227611940298508, + "recall": 0.06633018043080836, + "f1-score": 0.11989081244163494, + "support": 25162.0 + }, + "accuracy": 0.7758937635470683, + "macro avg": { + "precision": 0.701251308896494, + "recall": 0.5271600295708253, + "f1-score": 0.4957450289414211, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7436164885554023, + "recall": 0.7758937635470683, + "f1-score": 0.6986130134592862, + "support": 109341.0 + }, + "cross-entropy": 0.5001025680620461, + "roc-auc": 0.6830259884114023, + "score": 0.6830259884114023 + }, + "val": { + "0": { + "precision": 0.6732070365358592, + "recall": 0.9617863126691584, + "f1-score": 0.7920292931437062, + "support": 15518.0 + }, + "1": { + "precision": 0.7164036346245816, + "recall": 0.17133706965572457, + "f1-score": 0.2765368285028614, + "support": 8743.0 + }, + "accuracy": 0.6769300523473888, + "macro avg": { + "precision": 0.6948053355802204, + "recall": 0.5665616911624415, + "f1-score": 0.5342830608232838, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6887739075259132, + "recall": 0.6769300523473888, + "f1-score": 0.6062599259142059, + "support": 24261.0 + }, + "cross-entropy": 0.6465081837078666, + "roc-auc": 0.639398116545268, + "score": 0.639398116545268 + }, + "test": { + "0": { + "precision": 0.6569128900519409, + "recall": 0.979080486940078, + "f1-score": 0.7862753007616924, + "support": 16922.0 + }, + "1": { + "precision": 0.713128038897893, + "recall": 0.09231091996223645, + "f1-score": 0.16346243150366863, + "support": 9533.0 + }, + "accuracy": 0.6595350595350595, + "macro avg": { + "precision": 0.6850204644749169, + "recall": 0.5356957034511572, + "f1-score": 0.47486886613268053, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6771698930361957, + "recall": 0.6595350595350595, + "f1-score": 0.5618460782088011, + "support": 26455.0 + }, + "cross-entropy": 0.6832483314751614, + "roc-auc": 0.6013710911801927, + "score": 0.6013710911801927 + } + }, + "time": "0:02:55.144244", + "diversity": { + "train": { + "mean_centered_corr": -0.016687747091054916, + "mean_pairwise_disagreement": 0.024748319920131486, + "member_std": 0.02787911146879196 + }, + "val": { + "mean_centered_corr": 0.00230225152336061, + "mean_pairwise_disagreement": 0.0672119464266957, + "member_std": 0.058351658284664154 + }, + "test": { + "mean_centered_corr": 0.008990149945020676, + "mean_pairwise_disagreement": 0.0442228434163918, + "member_std": 0.059662897139787674 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 11 + ], + "head_selection_time": "0:00:11.157803" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..29cd98b02 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.001-best-head-evaluation/0/report.json @@ -0,0 +1,228 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 2457536, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7796848489393414, + "recall": 0.9880849142897873, + "f1-score": 0.8716008760439699, + "support": 84179.0 + }, + "1": { + "precision": 0.6232156273478587, + "recall": 0.06593275574278674, + "f1-score": 0.11924956871765383, + "support": 25162.0 + }, + "accuracy": 0.7758754721467702, + "macro avg": { + "precision": 0.7014502381436001, + "recall": 0.527008835016287, + "f1-score": 0.4954252223808118, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7436775090239859, + "recall": 0.7758754721467702, + "f1-score": 0.6984667031815965, + "support": 109341.0 + }, + "cross-entropy": 0.5001057107203039, + "roc-auc": 0.6829994192308994, + "score": 0.6829994192308994 + }, + "val": { + "0": { + "precision": 0.6728264298913779, + "recall": 0.961979636551102, + "f1-score": 0.7918313221058215, + "support": 15518.0 + }, + "1": { + "precision": 0.7155255544840887, + "recall": 0.16973578863090472, + "f1-score": 0.27438291578071555, + "support": 8743.0 + }, + "accuracy": 0.676476649767116, + "macro avg": { + "precision": 0.6941759921877333, + "recall": 0.5658577125910034, + "f1-score": 0.5331071189432686, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6882140250570375, + "recall": 0.676476649767116, + "f1-score": 0.6053570870577856, + "support": 24261.0 + }, + "cross-entropy": 0.6469691623792566, + "roc-auc": 0.6392187010153481, + "score": 0.6392187010153481 + }, + "test": { + "0": { + "precision": 0.6566040725774502, + "recall": 0.9794350549580428, + "f1-score": 0.7861682952281567, + "support": 16922.0 + }, + "1": { + "precision": 0.7131079967023908, + "recall": 0.09073743837197105, + "f1-score": 0.16099013586450772, + "support": 9533.0 + }, + "accuracy": 0.6591948591948592, + "macro avg": { + "precision": 0.6848560346399205, + "recall": 0.5350862466650069, + "f1-score": 0.4735792155463322, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6769651350867323, + "recall": 0.6591948591948592, + "f1-score": 0.5608867456831306, + "support": 26455.0 + }, + "cross-entropy": 0.6838126131386375, + "roc-auc": 0.6012738729168664, + "score": 0.6012738729168664 + } + }, + "time": "0:02:54.252595", + "diversity": { + "train": { + "mean_centered_corr": -0.01680082082748413, + "mean_pairwise_disagreement": 0.024706850807963605, + "member_std": 0.027859199792146683 + }, + "val": { + "mean_centered_corr": 0.002759142080321908, + "mean_pairwise_disagreement": 0.06733643269232048, + "member_std": 0.0583425834774971 + }, + "test": { + "mean_centered_corr": 0.009294640272855759, + "mean_pairwise_disagreement": 0.04442342748794362, + "member_std": 0.05963234603404999 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 11 + ], + "head_selection_time": "0:00:04.876008" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.001-evaluation/0.toml new file mode 100644 index 000000000..6054af7db --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.001-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 8 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.001-evaluation/0/report.json new file mode 100644 index 000000000..5cd64046b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.001-evaluation/0/report.json @@ -0,0 +1,224 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 2457536, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7813927209378972, + "recall": 0.9857565426056379, + "f1-score": 0.8717576980049797, + "support": 84179.0 + }, + "1": { + "precision": 0.6188811188811189, + "recall": 0.0773785867578094, + "f1-score": 0.13755828740991946, + "support": 25162.0 + }, + "accuracy": 0.776716876560485, + "macro avg": { + "precision": 0.700136919909508, + "recall": 0.5315675646817237, + "f1-score": 0.5046579927074496, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7439948836128988, + "recall": 0.776716876560485, + "f1-score": 0.7028007141709842, + "support": 109341.0 + }, + "cross-entropy": 0.4956186570036448, + "roc-auc": 0.69174046857932, + "score": 0.69174046857932 + }, + "val": { + "0": { + "precision": 0.6748435657930534, + "recall": 0.9590797783219487, + "f1-score": 0.7922389013094858, + "support": 15518.0 + }, + "1": { + "precision": 0.712279111916629, + "recall": 0.1798009836440581, + "f1-score": 0.28712328767123285, + "support": 8743.0 + }, + "accuracy": 0.678249041671819, + "macro avg": { + "precision": 0.6935613388548412, + "recall": 0.5694403809830034, + "f1-score": 0.5396810944903594, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6883343114242483, + "recall": 0.678249041671819, + "f1-score": 0.610209067005902, + "support": 24261.0 + }, + "cross-entropy": 0.6759987255155949, + "roc-auc": 0.6225384372823319, + "score": 0.6225384372823319 + }, + "test": { + "0": { + "precision": 0.6569276511397423, + "recall": 0.9792577709490604, + "f1-score": 0.7863430374641137, + "support": 16922.0 + }, + "1": { + "precision": 0.7146341463414634, + "recall": 0.09220602118955208, + "f1-score": 0.1633373594722661, + "support": 9533.0 + }, + "accuracy": 0.6596106596106596, + "macro avg": { + "precision": 0.6857808987406029, + "recall": 0.5357318960693063, + "f1-score": 0.4748401984681899, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6777220574432014, + "recall": 0.6596106596106596, + "f1-score": 0.5618443367158135, + "support": 26455.0 + }, + "cross-entropy": 0.7377377305054865, + "roc-auc": 0.591035974005685, + "score": 0.591035974005685 + } + }, + "time": "0:02:54.252595", + "diversity": { + "train": { + "mean_centered_corr": -0.01680082082748413, + "mean_pairwise_disagreement": 0.024706850807963605, + "member_std": 0.027859199792146683 + }, + "val": { + "mean_centered_corr": 0.002759142080321908, + "mean_pairwise_disagreement": 0.06733643269232048, + "member_std": 0.0583425834774971 + }, + "test": { + "mean_centered_corr": 0.009294640272855759, + "mean_pairwise_disagreement": 0.04442342748794362, + "member_std": 0.05963234603404999 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..7e28a06fe --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,228 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 2457536, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7796848489393414, + "recall": 0.9880849142897873, + "f1-score": 0.8716008760439699, + "support": 84179.0 + }, + "1": { + "precision": 0.6232156273478587, + "recall": 0.06593275574278674, + "f1-score": 0.11924956871765383, + "support": 25162.0 + }, + "accuracy": 0.7758754721467702, + "macro avg": { + "precision": 0.7014502381436001, + "recall": 0.527008835016287, + "f1-score": 0.4954252223808118, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7436775090239859, + "recall": 0.7758754721467702, + "f1-score": 0.6984667031815965, + "support": 109341.0 + }, + "cross-entropy": 0.5001057107203039, + "roc-auc": 0.6829994192308994, + "score": 0.6829994192308994 + }, + "val": { + "0": { + "precision": 0.6728264298913779, + "recall": 0.961979636551102, + "f1-score": 0.7918313221058215, + "support": 15518.0 + }, + "1": { + "precision": 0.7155255544840887, + "recall": 0.16973578863090472, + "f1-score": 0.27438291578071555, + "support": 8743.0 + }, + "accuracy": 0.676476649767116, + "macro avg": { + "precision": 0.6941759921877333, + "recall": 0.5658577125910034, + "f1-score": 0.5331071189432686, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6882140250570375, + "recall": 0.676476649767116, + "f1-score": 0.6053570870577856, + "support": 24261.0 + }, + "cross-entropy": 0.6469691623792566, + "roc-auc": 0.6392187010153481, + "score": 0.6392187010153481 + }, + "test": { + "0": { + "precision": 0.6566040725774502, + "recall": 0.9794350549580428, + "f1-score": 0.7861682952281567, + "support": 16922.0 + }, + "1": { + "precision": 0.7131079967023908, + "recall": 0.09073743837197105, + "f1-score": 0.16099013586450772, + "support": 9533.0 + }, + "accuracy": 0.6591948591948592, + "macro avg": { + "precision": 0.6848560346399205, + "recall": 0.5350862466650069, + "f1-score": 0.4735792155463322, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6769651350867323, + "recall": 0.6591948591948592, + "f1-score": 0.5608867456831306, + "support": 26455.0 + }, + "cross-entropy": 0.6838126131386375, + "roc-auc": 0.6012738729168664, + "score": 0.6012738729168664 + } + }, + "time": "0:02:54.252595", + "diversity": { + "train": { + "mean_centered_corr": -0.01680082082748413, + "mean_pairwise_disagreement": 0.024706850807963605, + "member_std": 0.027859199792146683 + }, + "val": { + "mean_centered_corr": 0.002759142080321908, + "mean_pairwise_disagreement": 0.06733643269232048, + "member_std": 0.0583425834774971 + }, + "test": { + "mean_centered_corr": 0.009294640272855759, + "mean_pairwise_disagreement": 0.04442342748794362, + "member_std": 0.05963234603404999 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 11 + ], + "head_selection_time": "0:00:11.186604" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise1e-05-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise1e-05-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise1e-05-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise1e-05-best-head-evaluation/0/report.json new file mode 100644 index 000000000..4cf780715 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise1e-05-best-head-evaluation/0/report.json @@ -0,0 +1,228 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 2457536, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7796294039908898, + "recall": 0.9881443115266277, + "f1-score": 0.8715893373569722, + "support": 84179.0 + }, + "1": { + "precision": 0.6231117824773413, + "recall": 0.06557507352356728, + "f1-score": 0.1186623516720604, + "support": 25162.0 + }, + "accuracy": 0.7758388893461738, + "macro avg": { + "precision": 0.7013705932341155, + "recall": 0.5268596925250975, + "f1-score": 0.4951258445145163, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7436109260866827, + "recall": 0.7758388893461738, + "f1-score": 0.6983226870263208, + "support": 109341.0 + }, + "cross-entropy": 0.5001170885588134, + "roc-auc": 0.6829371288514838, + "score": 0.6829371288514838 + }, + "val": { + "0": { + "precision": 0.673176672229489, + "recall": 0.9617863126691584, + "f1-score": 0.7920082782774815, + "support": 15518.0 + }, + "1": { + "precision": 0.716267942583732, + "recall": 0.171222692439666, + "f1-score": 0.2763777346995292, + "support": 8743.0 + }, + "accuracy": 0.6768888339310004, + "macro avg": { + "precision": 0.6947223074066105, + "recall": 0.5665045025544122, + "f1-score": 0.5341930064885054, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.688705585988491, + "recall": 0.6768888339310004, + "f1-score": 0.6061891511804106, + "support": 24261.0 + }, + "cross-entropy": 0.6463251283606344, + "roc-auc": 0.6393903110631306, + "score": 0.6393903110631306 + }, + "test": { + "0": { + "precision": 0.6568744053282588, + "recall": 0.9791395816097388, + "f1-score": 0.78626678688369, + "support": 16922.0 + }, + "1": { + "precision": 0.7132412672623883, + "recall": 0.09210112241686773, + "f1-score": 0.16313638052768487, + "support": 9533.0 + }, + "accuracy": 0.6594972594972595, + "macro avg": { + "precision": 0.6850578362953236, + "recall": 0.5356203520133033, + "f1-score": 0.47470158370568744, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6771860777840538, + "recall": 0.6594972594972595, + "f1-score": 0.5617231405487138, + "support": 26455.0 + }, + "cross-entropy": 0.6828939339210119, + "roc-auc": 0.6013715932958167, + "score": 0.6013715932958167 + } + }, + "time": "0:02:54.360546", + "diversity": { + "train": { + "mean_centered_corr": -0.01661285012960434, + "mean_pairwise_disagreement": 0.024667170270220036, + "member_std": 0.027912361547350883 + }, + "val": { + "mean_centered_corr": 0.0025234981440007687, + "mean_pairwise_disagreement": 0.06728490967183492, + "member_std": 0.058532536029815674 + }, + "test": { + "mean_centered_corr": 0.009481742046773434, + "mean_pairwise_disagreement": 0.04442228434163918, + "member_std": 0.05997689068317413 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 11 + ], + "head_selection_time": "0:00:04.892264" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise1e-05-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise1e-05-evaluation/0.toml new file mode 100644 index 000000000..7d7b01333 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise1e-05-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 8 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise1e-05-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise1e-05-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise1e-05-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise1e-05-evaluation/0/report.json new file mode 100644 index 000000000..1992163e5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise1e-05-evaluation/0/report.json @@ -0,0 +1,224 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 2457536, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7814062544139665, + "recall": 0.9857921809477423, + "f1-score": 0.8717800563096189, + "support": 84179.0 + }, + "1": { + "precision": 0.6195928753180662, + "recall": 0.07741832922661156, + "f1-score": 0.13763866318095103, + "support": 25162.0 + }, + "accuracy": 0.7767534593610814, + "macro avg": { + "precision": 0.7004995648660164, + "recall": 0.5316052550871769, + "f1-score": 0.504709359745285, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7441690950244325, + "recall": 0.7767534593610814, + "f1-score": 0.702836423693276, + "support": 109341.0 + }, + "cross-entropy": 0.4956384516885499, + "roc-auc": 0.6916909499513633, + "score": 0.6916909499513633 + }, + "val": { + "0": { + "precision": 0.6749659863945578, + "recall": 0.9590797783219487, + "f1-score": 0.7923232538330494, + "support": 15518.0 + }, + "1": { + "precision": 0.7127996381727725, + "recall": 0.18025849250829235, + "f1-score": 0.28774876757348916, + "support": 8743.0 + }, + "accuracy": 0.6784139153373727, + "macro avg": { + "precision": 0.6938828122836651, + "recall": 0.5696691354151205, + "f1-score": 0.5400360107032692, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6886001984013561, + "recall": 0.6784139153373727, + "f1-score": 0.6104884270176941, + "support": 24261.0 + }, + "cross-entropy": 0.6754694410618766, + "roc-auc": 0.6226251931156621, + "score": 0.6226251931156621 + }, + "test": { + "0": { + "precision": 0.6569276511397423, + "recall": 0.9792577709490604, + "f1-score": 0.7863430374641137, + "support": 16922.0 + }, + "1": { + "precision": 0.7146341463414634, + "recall": 0.09220602118955208, + "f1-score": 0.1633373594722661, + "support": 9533.0 + }, + "accuracy": 0.6596106596106596, + "macro avg": { + "precision": 0.6857808987406029, + "recall": 0.5357318960693063, + "f1-score": 0.4748401984681899, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6777220574432014, + "recall": 0.6596106596106596, + "f1-score": 0.5618443367158135, + "support": 26455.0 + }, + "cross-entropy": 0.736898658236868, + "roc-auc": 0.5910938288836818, + "score": 0.5910938288836818 + } + }, + "time": "0:02:54.360546", + "diversity": { + "train": { + "mean_centered_corr": -0.01661285012960434, + "mean_pairwise_disagreement": 0.024667170270220036, + "member_std": 0.027912361547350883 + }, + "val": { + "mean_centered_corr": 0.0025234981440007687, + "mean_pairwise_disagreement": 0.06728490967183492, + "member_std": 0.058532536029815674 + }, + "test": { + "mean_centered_corr": 0.009481742046773434, + "mean_pairwise_disagreement": 0.04442228434163918, + "member_std": 0.05997689068317413 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise1e-05-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise1e-05-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise1e-05-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise1e-05-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..3dc14a84b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise1e-05-greedy-heads-evaluation/0/report.json @@ -0,0 +1,228 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 2457536, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7796294039908898, + "recall": 0.9881443115266277, + "f1-score": 0.8715893373569722, + "support": 84179.0 + }, + "1": { + "precision": 0.6231117824773413, + "recall": 0.06557507352356728, + "f1-score": 0.1186623516720604, + "support": 25162.0 + }, + "accuracy": 0.7758388893461738, + "macro avg": { + "precision": 0.7013705932341155, + "recall": 0.5268596925250975, + "f1-score": 0.4951258445145163, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7436109260866827, + "recall": 0.7758388893461738, + "f1-score": 0.6983226870263208, + "support": 109341.0 + }, + "cross-entropy": 0.5001170885588134, + "roc-auc": 0.6829371288514838, + "score": 0.6829371288514838 + }, + "val": { + "0": { + "precision": 0.673176672229489, + "recall": 0.9617863126691584, + "f1-score": 0.7920082782774815, + "support": 15518.0 + }, + "1": { + "precision": 0.716267942583732, + "recall": 0.171222692439666, + "f1-score": 0.2763777346995292, + "support": 8743.0 + }, + "accuracy": 0.6768888339310004, + "macro avg": { + "precision": 0.6947223074066105, + "recall": 0.5665045025544122, + "f1-score": 0.5341930064885054, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.688705585988491, + "recall": 0.6768888339310004, + "f1-score": 0.6061891511804106, + "support": 24261.0 + }, + "cross-entropy": 0.6463251283606344, + "roc-auc": 0.6393903110631306, + "score": 0.6393903110631306 + }, + "test": { + "0": { + "precision": 0.6568744053282588, + "recall": 0.9791395816097388, + "f1-score": 0.78626678688369, + "support": 16922.0 + }, + "1": { + "precision": 0.7132412672623883, + "recall": 0.09210112241686773, + "f1-score": 0.16313638052768487, + "support": 9533.0 + }, + "accuracy": 0.6594972594972595, + "macro avg": { + "precision": 0.6850578362953236, + "recall": 0.5356203520133033, + "f1-score": 0.47470158370568744, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6771860777840538, + "recall": 0.6594972594972595, + "f1-score": 0.5617231405487138, + "support": 26455.0 + }, + "cross-entropy": 0.6828939339210119, + "roc-auc": 0.6013715932958167, + "score": 0.6013715932958167 + } + }, + "time": "0:02:54.360546", + "diversity": { + "train": { + "mean_centered_corr": -0.01661285012960434, + "mean_pairwise_disagreement": 0.024667170270220036, + "member_std": 0.027912361547350883 + }, + "val": { + "mean_centered_corr": 0.0025234981440007687, + "mean_pairwise_disagreement": 0.06728490967183492, + "member_std": 0.058532536029815674 + }, + "test": { + "mean_centered_corr": 0.009481742046773434, + "mean_pairwise_disagreement": 0.04442228434163918, + "member_std": 0.05997689068317413 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 11 + ], + "head_selection_time": "0:00:11.225895" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_four_combined-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_four_combined-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_four_combined-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_four_combined-best-head-evaluation/0/report.json new file mode 100644 index 000000000..5d789d8e2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_four_combined-best-head-evaluation/0/report.json @@ -0,0 +1,680 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.901321291923523, + "min_features_per_member": 256.0, + "mean_features_per_member": 268.59375, + "max_features_per_member": 279.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.842281341552734, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8197222809700668, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_four_combined-evaluation/_esam_diagnostics/all_four_combined-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9513552811475944, + "recall": 0.9636150815239791, + "f1-score": 0.9574459372022485, + "support": 181971.0 + }, + "1": { + "precision": 0.8344915508449156, + "recall": 0.7882830763418263, + "f1-score": 0.8107294209075565, + "support": 42349.0 + }, + "accuracy": 0.9305144436519258, + "macro avg": { + "precision": 0.8929234159962549, + "recall": 0.8759490789329027, + "f1-score": 0.8840876790549025, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9292927717209353, + "recall": 0.9305144436519258, + "f1-score": 0.9297475699208473, + "support": 224320.0 + }, + "cross-entropy": 0.15873224362284377, + "roc-auc": 0.9722680689598284, + "score": 0.9722680689598284 + }, + "val": { + "0": { + "precision": 0.9436578686709782, + "recall": 0.9674451944328296, + "f1-score": 0.9554034922133081, + "support": 16741.0 + }, + "1": { + "precision": 0.8168067226890756, + "recall": 0.7153370621136297, + "f1-score": 0.7627118644067796, + "support": 3397.0 + }, + "accuracy": 0.9249180653490913, + "macro avg": { + "precision": 0.8802322956800269, + "recall": 0.8413911282732296, + "f1-score": 0.8590576783100439, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9222598478694823, + "recall": 0.9249180653490913, + "f1-score": 0.9228990995398164, + "support": 20138.0 + }, + "cross-entropy": 0.17990073731774503, + "roc-auc": 0.9569685349939213, + "score": 0.9569685349939213 + }, + "test": { + "0": { + "precision": 0.9437623162043353, + "recall": 0.9471362287974443, + "f1-score": 0.9454462624805436, + "support": 13147.0 + }, + "1": { + "precision": 0.7758787487907127, + "recall": 0.764294790343075, + "f1-score": 0.7700432069131061, + "support": 3148.0 + }, + "accuracy": 0.9118134397054312, + "macro avg": { + "precision": 0.859820532497524, + "recall": 0.8557155095702597, + "f1-score": 0.8577447346968249, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9113292097165732, + "recall": 0.9118134397054312, + "f1-score": 0.9115604804046742, + "support": 16295.0 + }, + "cross-entropy": 0.19410702240157, + "roc-auc": 0.9582203301945192, + "score": 0.9582203301945192 + } + }, + "time": "0:14:29.029077", + "diversity": { + "train": { + "mean_centered_corr": 0.09211483597755432, + "mean_pairwise_disagreement": 0.03578823167364594, + "member_std": 0.03491802513599396 + }, + "val": { + "mean_centered_corr": 0.1013437807559967, + "mean_pairwise_disagreement": 0.03309443068632885, + "member_std": 0.03697067126631737 + }, + "test": { + "mean_centered_corr": 0.12410959601402283, + "mean_pairwise_disagreement": 0.0401679220817785, + "member_std": 0.03913382440805435 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 6 + ], + "head_selection_time": "0:00:01.134116" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_four_combined-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_four_combined-evaluation/0.toml new file mode 100644 index 000000000..5b9249ca9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_four_combined-evaluation/0.toml @@ -0,0 +1,75 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 15 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "homesite-insurance" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_four_combined-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_four_combined-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_four_combined-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_four_combined-evaluation/0/report.json new file mode 100644 index 000000000..0f201487a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_four_combined-evaluation/0/report.json @@ -0,0 +1,676 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.901321291923523, + "min_features_per_member": 256.0, + "mean_features_per_member": 268.59375, + "max_features_per_member": 279.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.842281341552734, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8197222809700668, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_four_combined-evaluation/_esam_diagnostics/all_four_combined-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9439755183585884, + "recall": 0.9814915563468904, + "f1-score": 0.9623680537968715, + "support": 181971.0 + }, + "1": { + "precision": 0.9040920351966284, + "recall": 0.7496989303171268, + "f1-score": 0.8196886375958485, + "support": 42349.0 + }, + "accuracy": 0.9377318116975749, + "macro avg": { + "precision": 0.9240337767776083, + "recall": 0.8655952433320087, + "f1-score": 0.89102834569636, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9364459818552635, + "recall": 0.9377318116975749, + "f1-score": 0.9354318439328508, + "support": 224320.0 + }, + "cross-entropy": 0.14691183931212085, + "roc-auc": 0.9764623137919726, + "score": 0.9764623137919726 + }, + "val": { + "0": { + "precision": 0.934692250754427, + "recall": 0.980586583836091, + "f1-score": 0.957089552238806, + "support": 16741.0 + }, + "1": { + "precision": 0.8737864077669902, + "recall": 0.6623491315866942, + "f1-score": 0.753516409912927, + "support": 3397.0 + }, + "accuracy": 0.9269043599165756, + "macro avg": { + "precision": 0.9042393292607086, + "recall": 0.8214678577113925, + "f1-score": 0.8553029810758666, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.924418283695716, + "recall": 0.9269043599165756, + "f1-score": 0.9227495996873605, + "support": 20138.0 + }, + "cross-entropy": 0.17471226841771598, + "roc-auc": 0.958474253988237, + "score": 0.958474253988237 + }, + "test": { + "0": { + "precision": 0.9308116741687237, + "recall": 0.9752034684718948, + "f1-score": 0.9524906207050258, + "support": 13147.0 + }, + "1": { + "precision": 0.8706862356207854, + "recall": 0.6972681067344345, + "f1-score": 0.7743870171106015, + "support": 3148.0 + }, + "accuracy": 0.9215096655415772, + "macro avg": { + "precision": 0.9007489548947545, + "recall": 0.8362357876031646, + "f1-score": 0.8634388189078137, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9191961552642187, + "recall": 0.9215096655415772, + "f1-score": 0.9180831249016966, + "support": 16295.0 + }, + "cross-entropy": 0.17874492250186327, + "roc-auc": 0.962487069534998, + "score": 0.962487069534998 + } + }, + "time": "0:14:29.029077", + "diversity": { + "train": { + "mean_centered_corr": 0.09211483597755432, + "mean_pairwise_disagreement": 0.03578823167364594, + "member_std": 0.03491802513599396 + }, + "val": { + "mean_centered_corr": 0.1013437807559967, + "mean_pairwise_disagreement": 0.03309443068632885, + "member_std": 0.03697067126631737 + }, + "test": { + "mean_centered_corr": 0.12410959601402283, + "mean_pairwise_disagreement": 0.0401679220817785, + "member_std": 0.03913382440805435 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_four_combined-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_four_combined-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_four_combined-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_four_combined-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..6d2ffd9c7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_four_combined-greedy-heads-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.901321291923523, + "min_features_per_member": 256.0, + "mean_features_per_member": 268.59375, + "max_features_per_member": 279.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.842281341552734, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8197222809700668, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_four_combined-evaluation/_esam_diagnostics/all_four_combined-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9442336152219873, + "recall": 0.9817443438789697, + "f1-score": 0.9626236963555881, + "support": 181971.0 + }, + "1": { + "precision": 0.9054100227790433, + "recall": 0.7508559824316985, + "f1-score": 0.8209219171539583, + "support": 42349.0 + }, + "accuracy": 0.9381553138373752, + "macro avg": { + "precision": 0.9248218190005153, + "recall": 0.8663001631553341, + "f1-score": 0.8917728067547732, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9369041737260607, + "recall": 0.9381553138373752, + "f1-score": 0.9358720529559367, + "support": 224320.0 + }, + "cross-entropy": 0.14721044455331747, + "roc-auc": 0.9763538308497102, + "score": 0.9763538308497102 + }, + "val": { + "0": { + "precision": 0.9343600818367811, + "recall": 0.9820799235410071, + "f1-score": 0.9576258846142645, + "support": 16741.0 + }, + "1": { + "precision": 0.8819826907946499, + "recall": 0.6599941124521637, + "f1-score": 0.7550092608183195, + "support": 3397.0 + }, + "accuracy": 0.9277485351077565, + "macro avg": { + "precision": 0.9081713863157155, + "recall": 0.8210370179965853, + "f1-score": 0.8563175727162919, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9255247457870185, + "recall": 0.9277485351077565, + "f1-score": 0.9234472834108269, + "support": 20138.0 + }, + "cross-entropy": 0.17362982513958533, + "roc-auc": 0.9592737644154757, + "score": 0.9592737644154757 + }, + "test": { + "0": { + "precision": 0.9306966618287373, + "recall": 0.975507720392485, + "f1-score": 0.9525754818583578, + "support": 13147.0 + }, + "1": { + "precision": 0.8719681908548708, + "recall": 0.6966327827191868, + "f1-score": 0.7745011478015186, + "support": 3148.0 + }, + "accuracy": 0.9216324025774778, + "macro avg": { + "precision": 0.9013324263418041, + "recall": 0.8360702515558359, + "f1-score": 0.8635383148299383, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.919351020427956, + "recall": 0.9216324025774778, + "f1-score": 0.9181736405812219, + "support": 16295.0 + }, + "cross-entropy": 0.17835468537288718, + "roc-auc": 0.9627526496640615, + "score": 0.9627526496640615 + } + }, + "time": "0:14:29.029077", + "diversity": { + "train": { + "mean_centered_corr": 0.09211483597755432, + "mean_pairwise_disagreement": 0.03578823167364594, + "member_std": 0.03491802513599396 + }, + "val": { + "mean_centered_corr": 0.1013437807559967, + "mean_pairwise_disagreement": 0.03309443068632885, + "member_std": 0.03697067126631737 + }, + "test": { + "mean_centered_corr": 0.12410959601402283, + "mean_pairwise_disagreement": 0.0401679220817785, + "member_std": 0.03913382440805435 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 6, + 31, + 30, + 28, + 8, + 20, + 12, + 30, + 8, + 21, + 15 + ], + "head_selection_time": "0:00:14.268703" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_cf_fisd-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_cf_fisd-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_cf_fisd-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_cf_fisd-best-head-evaluation/0/report.json new file mode 100644 index 000000000..f2506d24e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_cf_fisd-best-head-evaluation/0/report.json @@ -0,0 +1,321 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.901321291923523, + "min_features_per_member": 256.0, + "mean_features_per_member": 268.59375, + "max_features_per_member": 279.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.842281341552734, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8197222809700668, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_cf_fisd-evaluation/_esam_diagnostics/all_minus_cf_fisd-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9506353233351531, + "recall": 0.9760401382637893, + "f1-score": 0.9631702390151977, + "support": 181971.0 + }, + "1": { + "precision": 0.8836899109000693, + "recall": 0.7822144560674396, + "f1-score": 0.8298615895284023, + "support": 42349.0 + }, + "accuracy": 0.9394481098430814, + "macro avg": { + "precision": 0.9171626171176113, + "recall": 0.8791272971656144, + "f1-score": 0.8965159142718, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9379968101788881, + "recall": 0.9394481098430814, + "f1-score": 0.9380031206257705, + "support": 224320.0 + }, + "cross-entropy": 0.1407259896841775, + "roc-auc": 0.9773294079585453, + "score": 0.9773294079585453 + }, + "val": { + "0": { + "precision": 0.9386192203604538, + "recall": 0.9737172211934771, + "f1-score": 0.955846135803917, + "support": 16741.0 + }, + "1": { + "precision": 0.8412125586430892, + "recall": 0.6861937003238151, + "f1-score": 0.7558365758754864, + "support": 3397.0 + }, + "accuracy": 0.9252160095342139, + "macro avg": { + "precision": 0.8899158895017715, + "recall": 0.8299554607586461, + "f1-score": 0.8558413558397017, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9221880737791702, + "recall": 0.9252160095342139, + "f1-score": 0.9221073099484756, + "support": 20138.0 + }, + "cross-entropy": 0.18155767784095098, + "roc-auc": 0.9560637742304589, + "score": 0.9560637742304589 + }, + "test": { + "0": { + "precision": 0.9381688466111772, + "recall": 0.960219061382825, + "f1-score": 0.949065894823892, + "support": 13147.0 + }, + "1": { + "precision": 0.8157802042972878, + "recall": 0.735705209656925, + "f1-score": 0.7736762986470687, + "support": 3148.0 + }, + "accuracy": 0.916845658177355, + "macro avg": { + "precision": 0.8769745254542325, + "recall": 0.847962135519875, + "f1-score": 0.8613710967354804, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9145248180131947, + "recall": 0.916845658177355, + "f1-score": 0.9151827129420486, + "support": 16295.0 + }, + "cross-entropy": 0.1860567271079682, + "roc-auc": 0.9602624689888717, + "score": 0.9602624689888717 + } + }, + "time": "0:13:27.087006", + "diversity": { + "train": { + "mean_centered_corr": 0.08523335307836533, + "mean_pairwise_disagreement": 0.038875824714693294, + "member_std": 0.03867573291063309 + }, + "val": { + "mean_centered_corr": 0.09791728109121323, + "mean_pairwise_disagreement": 0.036375120539246936, + "member_std": 0.04205026477575302 + }, + "test": { + "mean_centered_corr": 0.11845514178276062, + "mean_pairwise_disagreement": 0.044347043918082925, + "member_std": 0.04443347081542015 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 31 + ], + "head_selection_time": "0:00:01.210122" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_cf_fisd-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_cf_fisd-evaluation/0.toml new file mode 100644 index 000000000..e9e4aca66 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_cf_fisd-evaluation/0.toml @@ -0,0 +1,64 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_cf_fisd-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_cf_fisd-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_cf_fisd-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_cf_fisd-evaluation/0/report.json new file mode 100644 index 000000000..80d109a53 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_cf_fisd-evaluation/0/report.json @@ -0,0 +1,317 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.901321291923523, + "min_features_per_member": 256.0, + "mean_features_per_member": 268.59375, + "max_features_per_member": 279.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.842281341552734, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8197222809700668, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_cf_fisd-evaluation/_esam_diagnostics/all_minus_cf_fisd-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.94720362658795, + "recall": 0.9874870171620753, + "f1-score": 0.9669259391790271, + "support": 181971.0 + }, + "1": { + "precision": 0.9342097659635943, + "recall": 0.7634891024581454, + "f1-score": 0.8402655959666836, + "support": 42349.0 + }, + "accuracy": 0.945198823109843, + "macro avg": { + "precision": 0.9407066962757722, + "recall": 0.8754880598101104, + "f1-score": 0.9035957675728554, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.944750537235325, + "recall": 0.945198823109843, + "f1-score": 0.9430139434822568, + "support": 224320.0 + }, + "cross-entropy": 0.13349501270533368, + "roc-auc": 0.9819246121431815, + "score": 0.9819246121431815 + }, + "val": { + "0": { + "precision": 0.9319724044333861, + "recall": 0.9844692670688728, + "f1-score": 0.9575018155410312, + "support": 16741.0 + }, + "1": { + "precision": 0.8940505297473512, + "recall": 0.6458639976449808, + "f1-score": 0.7499572722611519, + "support": 3397.0 + }, + "accuracy": 0.9273512761942596, + "macro avg": { + "precision": 0.9130114670903686, + "recall": 0.8151666323569269, + "f1-score": 0.8537295439010916, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9255755125718079, + "recall": 0.9273512761942596, + "f1-score": 0.9224919429855764, + "support": 20138.0 + }, + "cross-entropy": 0.1755486957932297, + "roc-auc": 0.9580700807398709, + "score": 0.9580700807398709 + }, + "test": { + "0": { + "precision": 0.9290154691340177, + "recall": 0.9775614208564691, + "f1-score": 0.9526703976872614, + "support": 13147.0 + }, + "1": { + "precision": 0.880130028443722, + "recall": 0.6880559085133418, + "f1-score": 0.7723301836334463, + "support": 3148.0 + }, + "accuracy": 0.9216324025774778, + "macro avg": { + "precision": 0.9045727487888698, + "recall": 0.8328086646849054, + "f1-score": 0.8625002906603538, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9195713839978991, + "recall": 0.9216324025774778, + "f1-score": 0.917830815371127, + "support": 16295.0 + }, + "cross-entropy": 0.17967216581259585, + "roc-auc": 0.9619997518046595, + "score": 0.9619997518046595 + } + }, + "time": "0:13:27.087006", + "diversity": { + "train": { + "mean_centered_corr": 0.08523335307836533, + "mean_pairwise_disagreement": 0.038875824714693294, + "member_std": 0.03867573291063309 + }, + "val": { + "mean_centered_corr": 0.09791728109121323, + "mean_pairwise_disagreement": 0.036375120539246936, + "member_std": 0.04205026477575302 + }, + "test": { + "mean_centered_corr": 0.11845514178276062, + "mean_pairwise_disagreement": 0.044347043918082925, + "member_std": 0.04443347081542015 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_cf_fisd-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_cf_fisd-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_cf_fisd-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_cf_fisd-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..7386df965 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_cf_fisd-greedy-heads-evaluation/0/report.json @@ -0,0 +1,333 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.901321291923523, + "min_features_per_member": 256.0, + "mean_features_per_member": 268.59375, + "max_features_per_member": 279.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.842281341552734, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8197222809700668, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_cf_fisd-evaluation/_esam_diagnostics/all_minus_cf_fisd-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9475126453288841, + "recall": 0.9861846118337537, + "f1-score": 0.9664619287016417, + "support": 181971.0 + }, + "1": { + "precision": 0.9280109959337953, + "recall": 0.765260100592694, + "f1-score": 0.8388140440786324, + "support": 42349.0 + }, + "accuracy": 0.9444766405135521, + "macro avg": { + "precision": 0.9377618206313397, + "recall": 0.8757223562132239, + "f1-score": 0.902637986390137, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9438309613496018, + "recall": 0.9444766405135521, + "f1-score": 0.9423634967031582, + "support": 224320.0 + }, + "cross-entropy": 0.1342408923499761, + "roc-auc": 0.9812870495057586, + "score": 0.9812870495057586 + }, + "val": { + "0": { + "precision": 0.9332577903682719, + "recall": 0.9839316647751031, + "f1-score": 0.9579250385275218, + "support": 16741.0 + }, + "1": { + "precision": 0.8918810289389068, + "recall": 0.6532234324403886, + "f1-score": 0.7541206457094307, + "support": 3397.0 + }, + "accuracy": 0.9281457940212533, + "macro avg": { + "precision": 0.9125694096535893, + "recall": 0.8185775486077458, + "f1-score": 0.8560228421184762, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9262781072529898, + "recall": 0.9281457940212533, + "f1-score": 0.9235460772402513, + "support": 20138.0 + }, + "cross-entropy": 0.17438799592759188, + "roc-auc": 0.9587104803714672, + "score": 0.9587104803714672 + }, + "test": { + "0": { + "precision": 0.9292336665218021, + "recall": 0.9758119723130753, + "f1-score": 0.9519534003635959, + "support": 13147.0 + }, + "1": { + "precision": 0.8722378465247087, + "recall": 0.6896442185514613, + "f1-score": 0.7702678729820827, + "support": 3148.0 + }, + "accuracy": 0.9205277692543725, + "macro avg": { + "precision": 0.9007357565232554, + "recall": 0.8327280954322682, + "f1-score": 0.8611106366728393, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9182227526616702, + "recall": 0.9205277692543725, + "f1-score": 0.916853919529168, + "support": 16295.0 + }, + "cross-entropy": 0.17983663015934195, + "roc-auc": 0.9619912225060596, + "score": 0.9619912225060596 + } + }, + "time": "0:13:27.087006", + "diversity": { + "train": { + "mean_centered_corr": 0.08523335307836533, + "mean_pairwise_disagreement": 0.038875824714693294, + "member_std": 0.03867573291063309 + }, + "val": { + "mean_centered_corr": 0.09791728109121323, + "mean_pairwise_disagreement": 0.036375120539246936, + "member_std": 0.04205026477575302 + }, + "test": { + "mean_centered_corr": 0.11845514178276062, + "mean_pairwise_disagreement": 0.044347043918082925, + "member_std": 0.04443347081542015 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 31, + 2, + 4, + 20, + 28, + 22, + 7, + 16, + 25, + 9, + 29, + 1, + 2 + ], + "head_selection_time": "0:00:13.607257" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_esam-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_esam-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_esam-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_esam-best-head-evaluation/0/report.json new file mode 100644 index 000000000..105ac355c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_esam-best-head-evaluation/0/report.json @@ -0,0 +1,670 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.901321291923523, + "min_features_per_member": 256.0, + "mean_features_per_member": 268.59375, + "max_features_per_member": 279.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.842281341552734, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8197222809700668, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1100, + "metrics": { + "train": { + "0": { + "precision": 0.9368912902937575, + "recall": 0.9795626775695029, + "f1-score": 0.9577519275716627, + "support": 181971.0 + }, + "1": { + "precision": 0.8908135404127888, + "recall": 0.7164750053129944, + "f1-score": 0.7941892422457794, + "support": 42349.0 + }, + "accuracy": 0.9298947931526391, + "macro avg": { + "precision": 0.9138524153532732, + "recall": 0.8480188414412486, + "f1-score": 0.875970584908721, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9281923484708743, + "recall": 0.9298947931526391, + "f1-score": 0.926873200035706, + "support": 224320.0 + }, + "cross-entropy": 0.1611006274052141, + "roc-auc": 0.9697728815477382, + "score": 0.9697728815477382 + }, + "val": { + "0": { + "precision": 0.9335682218559982, + "recall": 0.9813033868944507, + "f1-score": 0.9568408177529268, + "support": 16741.0 + }, + "1": { + "precision": 0.8768201495474223, + "recall": 0.6558728289667354, + "f1-score": 0.7504210171775009, + "support": 3397.0 + }, + "accuracy": 0.9264077862747045, + "macro avg": { + "precision": 0.9051941857017103, + "recall": 0.818588107930593, + "f1-score": 0.8536309174652139, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9239956127770316, + "recall": 0.9264077862747045, + "f1-score": 0.92202067361971, + "support": 20138.0 + }, + "cross-entropy": 0.17653579040347495, + "roc-auc": 0.9578664730808395, + "score": 0.9578664730808395 + }, + "test": { + "0": { + "precision": 0.9297203303869005, + "recall": 0.9760401612535179, + "f1-score": 0.952317340161045, + "support": 13147.0 + }, + "1": { + "precision": 0.8736462093862816, + "recall": 0.6918678526048284, + "f1-score": 0.7722035100159547, + "support": 3148.0 + }, + "accuracy": 0.9211414544338754, + "macro avg": { + "precision": 0.9016832698865911, + "recall": 0.8339540069291731, + "f1-score": 0.8622604250884998, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.918887477799607, + "recall": 0.9211414544338754, + "f1-score": 0.9175214925208642, + "support": 16295.0 + }, + "cross-entropy": 0.17954252670408766, + "roc-auc": 0.9618357621457454, + "score": 0.9618357621457454 + } + }, + "time": "0:07:00.265011", + "diversity": { + "train": { + "mean_centered_corr": 0.08075066655874252, + "mean_pairwise_disagreement": 0.03102939600973264, + "member_std": 0.0314626470208168 + }, + "val": { + "mean_centered_corr": 0.08396831154823303, + "mean_pairwise_disagreement": 0.02662335530004261, + "member_std": 0.032109007239341736 + }, + "test": { + "mean_centered_corr": 0.10440877079963684, + "mean_pairwise_disagreement": 0.03210229241108988, + "member_std": 0.03372453153133392 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 31 + ], + "head_selection_time": "0:00:01.603798" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_esam-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_esam-evaluation/0.toml new file mode 100644 index 000000000..294585115 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_esam-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 15 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "homesite-insurance" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_esam-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_esam-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_esam-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_esam-evaluation/0/report.json new file mode 100644 index 000000000..38c78f5bf --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_esam-evaluation/0/report.json @@ -0,0 +1,666 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.901321291923523, + "min_features_per_member": 256.0, + "mean_features_per_member": 268.59375, + "max_features_per_member": 279.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.842281341552734, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8197222809700668, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1100, + "metrics": { + "train": { + "0": { + "precision": 0.9351769160464812, + "recall": 0.9840194316676834, + "f1-score": 0.9589766659704482, + "support": 181971.0 + }, + "1": { + "precision": 0.9114629319531131, + "recall": 0.7069116153864318, + "f1-score": 0.7962603399207383, + "support": 42349.0 + }, + "accuracy": 0.9317047075606276, + "macro avg": { + "precision": 0.9233199239997971, + "recall": 0.8454655235270576, + "f1-score": 0.8776185029455932, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9306999924000384, + "recall": 0.9317047075606276, + "f1-score": 0.928257721195666, + "support": 224320.0 + }, + "cross-entropy": 0.15826633985081387, + "roc-auc": 0.9715808416321321, + "score": 0.9715808416321321 + }, + "val": { + "0": { + "precision": 0.9273451526032316, + "recall": 0.9873364793023117, + "f1-score": 0.9564009836539853, + "support": 16741.0 + }, + "1": { + "precision": 0.9083837510803803, + "recall": 0.6187812775978805, + "f1-score": 0.7361232708807565, + "support": 3397.0 + }, + "accuracy": 0.9251663521700269, + "macro avg": { + "precision": 0.917864451841806, + "recall": 0.803058878450096, + "f1-score": 0.8462621272673709, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9241466283717724, + "recall": 0.9251663521700269, + "f1-score": 0.9192432028271575, + "support": 20138.0 + }, + "cross-entropy": 0.1764182894477564, + "roc-auc": 0.95846243739381, + "score": 0.95846243739381 + }, + "test": { + "0": { + "precision": 0.9234446347031964, + "recall": 0.9844831520498973, + "f1-score": 0.9529875197879468, + "support": 13147.0 + }, + "1": { + "precision": 0.9104870557261957, + "recall": 0.659148665819568, + "f1-score": 0.7646950433020084, + "support": 3148.0 + }, + "accuracy": 0.9216324025774778, + "macro avg": { + "precision": 0.916965845214696, + "recall": 0.8218159089347327, + "f1-score": 0.8588412815449776, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.920941384711199, + "recall": 0.9216324025774778, + "f1-score": 0.9166116550455268, + "support": 16295.0 + }, + "cross-entropy": 0.18012155300228325, + "roc-auc": 0.9621719566520266, + "score": 0.9621719566520266 + } + }, + "time": "0:07:00.265011", + "diversity": { + "train": { + "mean_centered_corr": 0.08075066655874252, + "mean_pairwise_disagreement": 0.03102939600973264, + "member_std": 0.0314626470208168 + }, + "val": { + "mean_centered_corr": 0.08396831154823303, + "mean_pairwise_disagreement": 0.02662335530004261, + "member_std": 0.032109007239341736 + }, + "test": { + "mean_centered_corr": 0.10440877079963684, + "mean_pairwise_disagreement": 0.03210229241108988, + "member_std": 0.03372453153133392 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_esam-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_esam-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_esam-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_esam-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..f60bcb208 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_esam-greedy-heads-evaluation/0/report.json @@ -0,0 +1,678 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.901321291923523, + "min_features_per_member": 256.0, + "mean_features_per_member": 268.59375, + "max_features_per_member": 279.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.842281341552734, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8197222809700668, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1100, + "metrics": { + "train": { + "0": { + "precision": 0.9327077372080825, + "recall": 0.9869979282413132, + "f1-score": 0.9590851564877955, + "support": 181971.0 + }, + "1": { + "precision": 0.9254967408760273, + "recall": 0.6940187489669177, + "f1-score": 0.7932151242814347, + "support": 42349.0 + }, + "accuracy": 0.9316868758915835, + "macro avg": { + "precision": 0.9291022390420549, + "recall": 0.8405083386041154, + "f1-score": 0.876150140384615, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9313463851945921, + "recall": 0.9316868758915835, + "f1-score": 0.9277708287688797, + "support": 224320.0 + }, + "cross-entropy": 0.15549230413668702, + "roc-auc": 0.9720873725388706, + "score": 0.9720873725388706 + }, + "val": { + "0": { + "precision": 0.9275695691202872, + "recall": 0.9875754136550983, + "f1-score": 0.9566324316505135, + "support": 16741.0 + }, + "1": { + "precision": 0.9101123595505618, + "recall": 0.6199587871651457, + "f1-score": 0.7375240763438977, + "support": 3397.0 + }, + "accuracy": 0.9255636110835237, + "macro avg": { + "precision": 0.9188409643354245, + "recall": 0.803767100410122, + "f1-score": 0.8470782539972056, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9246247811121256, + "recall": 0.9255636110835237, + "f1-score": 0.9196719051346443, + "support": 20138.0 + }, + "cross-entropy": 0.1741207203692563, + "roc-auc": 0.959329480009883, + "score": 0.959329480009883 + }, + "test": { + "0": { + "precision": 0.9233678201926507, + "recall": 0.9843310260896022, + "f1-score": 0.9528753405492968, + "support": 13147.0 + }, + "1": { + "precision": 0.9096491228070176, + "recall": 0.658831003811944, + "f1-score": 0.7641857037582903, + "support": 3148.0 + }, + "accuracy": 0.9214482970236268, + "macro avg": { + "precision": 0.9165084714998342, + "recall": 0.8215810149507732, + "f1-score": 0.8585305221537936, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9207175311855951, + "recall": 0.9214482970236268, + "f1-score": 0.9164227491643266, + "support": 16295.0 + }, + "cross-entropy": 0.17746238406307344, + "roc-auc": 0.9630054358452254, + "score": 0.9630054358452254 + } + }, + "time": "0:07:00.265011", + "diversity": { + "train": { + "mean_centered_corr": 0.08075066655874252, + "mean_pairwise_disagreement": 0.03102939600973264, + "member_std": 0.0314626470208168 + }, + "val": { + "mean_centered_corr": 0.08396831154823303, + "mean_pairwise_disagreement": 0.02662335530004261, + "member_std": 0.032109007239341736 + }, + "test": { + "mean_centered_corr": 0.10440877079963684, + "mean_pairwise_disagreement": 0.03210229241108988, + "member_std": 0.03372453153133392 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 31, + 15, + 14, + 28, + 10, + 16, + 15, + 28, + 14 + ], + "head_selection_time": "0:00:19.762715" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_mfb-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_mfb-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_mfb-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_mfb-best-head-evaluation/0/report.json new file mode 100644 index 000000000..45764d1df --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_mfb-best-head-evaluation/0/report.json @@ -0,0 +1,644 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_mfb-evaluation/_esam_diagnostics/all_minus_mfb-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1100, + "metrics": { + "train": { + "0": { + "precision": 0.9369183420516376, + "recall": 0.9803375263091372, + "f1-score": 0.9581362873447465, + "support": 181971.0 + }, + "1": { + "precision": 0.8945040688760467, + "recall": 0.7163805520791519, + "f1-score": 0.7955943093162001, + "support": 42349.0 + }, + "accuracy": 0.9305055278174037, + "macro avg": { + "precision": 0.9157112054638421, + "recall": 0.8483590391941445, + "f1-score": 0.8768652983304733, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9289110219075885, + "recall": 0.9305055278174037, + "f1-score": 0.9274502574431286, + "support": 224320.0 + }, + "cross-entropy": 0.1606612384407506, + "roc-auc": 0.9700666591028972, + "score": 0.9700666591028972 + }, + "val": { + "0": { + "precision": 0.9329441106450516, + "recall": 0.9831551281285467, + "f1-score": 0.9573917342872932, + "support": 16741.0 + }, + "1": { + "precision": 0.8870192307692307, + "recall": 0.651751545481307, + "f1-score": 0.7513999660614288, + "support": 3397.0 + }, + "accuracy": 0.9272519614658854, + "macro avg": { + "precision": 0.9099816707071411, + "recall": 0.8174533368049268, + "f1-score": 0.8543958501743609, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9251972233206815, + "recall": 0.9272519614658854, + "f1-score": 0.9226437932473059, + "support": 20138.0 + }, + "cross-entropy": 0.1752607407742304, + "roc-auc": 0.9578946605821286, + "score": 0.9578946605821286 + }, + "test": { + "0": { + "precision": 0.9286075218364254, + "recall": 0.9784741766182399, + "f1-score": 0.9528888888888889, + "support": 13147.0 + }, + "1": { + "precision": 0.8841113841113841, + "recall": 0.6858322744599746, + "f1-score": 0.7724508050089446, + "support": 3148.0 + }, + "accuracy": 0.9219392451672292, + "macro avg": { + "precision": 0.9063594529739047, + "recall": 0.8321532255391073, + "f1-score": 0.8626698469489167, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9200113977763805, + "recall": 0.9219392451672292, + "f1-score": 0.9180303992875348, + "support": 16295.0 + }, + "cross-entropy": 0.18002408953514465, + "roc-auc": 0.9613487585255535, + "score": 0.9613487585255535 + } + }, + "time": "0:12:18.387954", + "diversity": { + "train": { + "mean_centered_corr": 0.018650218844413757, + "mean_pairwise_disagreement": 0.022819575146104644, + "member_std": 0.0206756591796875 + }, + "val": { + "mean_centered_corr": 0.01982957310974598, + "mean_pairwise_disagreement": 0.019789761132059755, + "member_std": 0.02188469097018242 + }, + "test": { + "mean_centered_corr": 0.02981637977063656, + "mean_pairwise_disagreement": 0.024840887269991785, + "member_std": 0.022842135280370712 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31 + ], + "head_selection_time": "0:00:01.553339" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_mfb-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_mfb-evaluation/0.toml new file mode 100644 index 000000000..6fe9e642b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_mfb-evaluation/0.toml @@ -0,0 +1,62 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "homesite-insurance" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_mfb-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_mfb-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_mfb-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_mfb-evaluation/0/report.json new file mode 100644 index 000000000..75867746e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_mfb-evaluation/0/report.json @@ -0,0 +1,640 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_mfb-evaluation/_esam_diagnostics/all_minus_mfb-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1100, + "metrics": { + "train": { + "0": { + "precision": 0.9361022029934762, + "recall": 0.983316022882767, + "f1-score": 0.9591284305317325, + "support": 181971.0 + }, + "1": { + "precision": 0.9084742696934069, + "recall": 0.7115870504616402, + "f1-score": 0.7980667372881356, + "support": 42349.0 + }, + "accuracy": 0.9320167617689016, + "macro avg": { + "precision": 0.9222882363434416, + "recall": 0.8474515366722035, + "f1-score": 0.878597583909934, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9308863713809378, + "recall": 0.9320167617689016, + "f1-score": 0.9287218611345628, + "support": 224320.0 + }, + "cross-entropy": 0.1556527280386954, + "roc-auc": 0.9717201052099171, + "score": 0.9717201052099171 + }, + "val": { + "0": { + "precision": 0.9289128845396771, + "recall": 0.986619676243952, + "f1-score": 0.9568970511557847, + "support": 16741.0 + }, + "1": { + "precision": 0.9049639372083157, + "recall": 0.627906976744186, + "f1-score": 0.7413972888425443, + "support": 3397.0 + }, + "accuracy": 0.9261098420895819, + "macro avg": { + "precision": 0.9169384108739964, + "recall": 0.807263326494069, + "f1-score": 0.8491471699991645, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9248730308260693, + "recall": 0.9261098420895819, + "f1-score": 0.9205452439962815, + "support": 20138.0 + }, + "cross-entropy": 0.17462322365346752, + "roc-auc": 0.9585091762449806, + "score": 0.9585091762449806 + }, + "test": { + "0": { + "precision": 0.924518852400372, + "recall": 0.9828858294667985, + "f1-score": 0.9528093201592686, + "support": 13147.0 + }, + "1": { + "precision": 0.9029335634167386, + "recall": 0.6648665819567979, + "f1-score": 0.765825100622027, + "support": 3148.0 + }, + "accuracy": 0.9214482970236268, + "macro avg": { + "precision": 0.9137262079085553, + "recall": 0.8238762057117982, + "f1-score": 0.8593172103906478, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9203488315522298, + "recall": 0.9214482970236268, + "f1-score": 0.9166861950838935, + "support": 16295.0 + }, + "cross-entropy": 0.17844908711477925, + "roc-auc": 0.9622794548091665, + "score": 0.9622794548091665 + } + }, + "time": "0:12:18.387954", + "diversity": { + "train": { + "mean_centered_corr": 0.018650218844413757, + "mean_pairwise_disagreement": 0.022819575146104644, + "member_std": 0.0206756591796875 + }, + "val": { + "mean_centered_corr": 0.01982957310974598, + "mean_pairwise_disagreement": 0.019789761132059755, + "member_std": 0.02188469097018242 + }, + "test": { + "mean_centered_corr": 0.02981637977063656, + "mean_pairwise_disagreement": 0.024840887269991785, + "member_std": 0.022842135280370712 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_mfb-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_mfb-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_mfb-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_mfb-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..04e352252 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_mfb-greedy-heads-evaluation/0/report.json @@ -0,0 +1,651 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_mfb-evaluation/_esam_diagnostics/all_minus_mfb-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1100, + "metrics": { + "train": { + "0": { + "precision": 0.9362959009617096, + "recall": 0.9833599859318243, + "f1-score": 0.9592510098126721, + "support": 181971.0 + }, + "1": { + "precision": 0.9088006746581532, + "recall": 0.7125079694916054, + "f1-score": 0.7987716906460537, + "support": 42349.0 + }, + "accuracy": 0.9322262838801711, + "macro avg": { + "precision": 0.9225482878099314, + "recall": 0.8479339777117149, + "f1-score": 0.8790113502293628, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9311051228824955, + "recall": 0.9322262838801711, + "f1-score": 0.928954385849641, + "support": 224320.0 + }, + "cross-entropy": 0.15555906132680442, + "roc-auc": 0.9717261720722767, + "score": 0.9717261720722767 + }, + "val": { + "0": { + "precision": 0.9296773830302348, + "recall": 0.9863210083029688, + "f1-score": 0.957161903657759, + "support": 16741.0 + }, + "1": { + "precision": 0.9036600757257046, + "recall": 0.6323226376214307, + "f1-score": 0.744024939383443, + "support": 3397.0 + }, + "accuracy": 0.926606415731453, + "macro avg": { + "precision": 0.9166687293779697, + "recall": 0.8093218229621997, + "f1-score": 0.850593421520601, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9252886258093842, + "recall": 0.926606415731453, + "f1-score": 0.9212086676045337, + "support": 20138.0 + }, + "cross-entropy": 0.17401332164145797, + "roc-auc": 0.9590512800985321, + "score": 0.9590512800985321 + }, + "test": { + "0": { + "precision": 0.9262719172175913, + "recall": 0.9804518141020765, + "f1-score": 0.9525920999150131, + "support": 13147.0 + }, + "1": { + "precision": 0.8919714165615805, + "recall": 0.6740787801778907, + "f1-score": 0.7678668355346481, + "support": 3148.0 + }, + "accuracy": 0.921264191469776, + "macro avg": { + "precision": 0.9091216668895858, + "recall": 0.8272652971399836, + "f1-score": 0.8602294677248306, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9196454688552026, + "recall": 0.921264191469776, + "f1-score": 0.916905378081973, + "support": 16295.0 + }, + "cross-entropy": 0.1777322749820877, + "roc-auc": 0.9625182147641627, + "score": 0.9625182147641627 + } + }, + "time": "0:12:18.387954", + "diversity": { + "train": { + "mean_centered_corr": 0.018650218844413757, + "mean_pairwise_disagreement": 0.022819575146104644, + "member_std": 0.0206756591796875 + }, + "val": { + "mean_centered_corr": 0.01982957310974598, + "mean_pairwise_disagreement": 0.019789761132059755, + "member_std": 0.02188469097018242 + }, + "test": { + "mean_centered_corr": 0.02981637977063656, + "mean_pairwise_disagreement": 0.024840887269991785, + "member_std": 0.022842135280370712 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31, + 15, + 5, + 21, + 14, + 3, + 15, + 2 + ], + "head_selection_time": "0:00:13.539201" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_rla-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_rla-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_rla-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_rla-best-head-evaluation/0/report.json new file mode 100644 index 000000000..6bd6b2bf1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_rla-best-head-evaluation/0/report.json @@ -0,0 +1,667 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.901321291923523, + "min_features_per_member": 256.0, + "mean_features_per_member": 268.59375, + "max_features_per_member": 279.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.842281341552734, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8197222809700668, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_rla-evaluation/_esam_diagnostics/all_minus_rla-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.941693557360632, + "recall": 0.976655620950591, + "f1-score": 0.9588559959859508, + "support": 181971.0 + }, + "1": { + "precision": 0.880650689742365, + "recall": 0.7401591536990247, + "f1-score": 0.8043160298683637, + "support": 42349.0 + }, + "accuracy": 0.9320078459343795, + "macro avg": { + "precision": 0.9111721235514985, + "recall": 0.8584073873248079, + "f1-score": 0.8815860129271573, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9301693758308264, + "recall": 0.9320078459343795, + "f1-score": 0.9296806526143669, + "support": 224320.0 + }, + "cross-entropy": 0.1563500457958513, + "roc-auc": 0.9712464870931182, + "score": 0.9712464870931182 + }, + "val": { + "0": { + "precision": 0.9341423118555233, + "recall": 0.9794516456603548, + "f1-score": 0.9562605703621625, + "support": 16741.0 + }, + "1": { + "precision": 0.8669245647969053, + "recall": 0.6596997350603474, + "f1-score": 0.7492477432296891, + "support": 3397.0 + }, + "accuracy": 0.9255139537193365, + "macro avg": { + "precision": 0.9005334383262142, + "recall": 0.819575690360351, + "f1-score": 0.8527541567959258, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9228036145291688, + "recall": 0.9255139537193365, + "f1-score": 0.9213403909119185, + "support": 20138.0 + }, + "cross-entropy": 0.1776733868160412, + "roc-auc": 0.9573359044742286, + "score": 0.9573359044742286 + }, + "test": { + "0": { + "precision": 0.9269699372792156, + "recall": 0.9780177987373545, + "f1-score": 0.9518099045081057, + "support": 13147.0 + }, + "1": { + "precision": 0.8807755775577558, + "recall": 0.6782083862770013, + "f1-score": 0.7663316582914573, + "support": 3148.0 + }, + "accuracy": 0.9200981896287205, + "macro avg": { + "precision": 0.9038727574184857, + "recall": 0.8281130925071779, + "f1-score": 0.8590707813997815, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.918045736947632, + "recall": 0.9200981896287205, + "f1-score": 0.9159777155489152, + "support": 16295.0 + }, + "cross-entropy": 0.18171269740302973, + "roc-auc": 0.9616503767533748, + "score": 0.9616503767533748 + } + }, + "time": "0:06:50.247903", + "diversity": { + "train": { + "mean_centered_corr": 0.08324653655290604, + "mean_pairwise_disagreement": 0.03557734342644149, + "member_std": 0.03504224866628647 + }, + "val": { + "mean_centered_corr": 0.09014386683702469, + "mean_pairwise_disagreement": 0.03127462845078635, + "member_std": 0.035533007234334946 + }, + "test": { + "mean_centered_corr": 0.11173245310783386, + "mean_pairwise_disagreement": 0.03804637777271872, + "member_std": 0.03772429749369621 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31 + ], + "head_selection_time": "0:00:01.240582" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_rla-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_rla-evaluation/0.toml new file mode 100644 index 000000000..98da74ee8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_rla-evaluation/0.toml @@ -0,0 +1,68 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 15 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "homesite-insurance" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_rla-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_rla-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_rla-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_rla-evaluation/0/report.json new file mode 100644 index 000000000..18547af5e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_rla-evaluation/0/report.json @@ -0,0 +1,663 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.901321291923523, + "min_features_per_member": 256.0, + "mean_features_per_member": 268.59375, + "max_features_per_member": 279.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.842281341552734, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8197222809700668, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_rla-evaluation/_esam_diagnostics/all_minus_rla-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9396844355041488, + "recall": 0.9857724582488419, + "f1-score": 0.9621768619910048, + "support": 181971.0 + }, + "1": { + "precision": 0.9225406893250359, + "recall": 0.728116366384094, + "f1-score": 0.8138782943792644, + "support": 42349.0 + }, + "accuracy": 0.9371299928673323, + "macro avg": { + "precision": 0.9311125624145924, + "recall": 0.856944412316468, + "f1-score": 0.8880275781851346, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.936447896154384, + "recall": 0.9371299928673323, + "f1-score": 0.9341798218706874, + "support": 224320.0 + }, + "cross-entropy": 0.1473269491192984, + "roc-auc": 0.9761188653178614, + "score": 0.9761188653178614 + }, + "val": { + "0": { + "precision": 0.9304999718166957, + "recall": 0.9860820739501822, + "f1-score": 0.9574850646714228, + "support": 16741.0 + }, + "1": { + "precision": 0.9027951606174385, + "recall": 0.6370326758904916, + "f1-score": 0.7469796341042457, + "support": 3397.0 + }, + "accuracy": 0.9272023041016982, + "macro avg": { + "precision": 0.9166475662170671, + "recall": 0.8115573749203369, + "f1-score": 0.8522323493878343, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9258265562022416, + "recall": 0.9272023041016982, + "f1-score": 0.921975731687179, + "support": 20138.0 + }, + "cross-entropy": 0.17509016585325612, + "roc-auc": 0.9586943908120914, + "score": 0.9586943908120914 + }, + "test": { + "0": { + "precision": 0.9261995266441942, + "recall": 0.982277325625618, + "f1-score": 0.9534145441122185, + "support": 13147.0 + }, + "1": { + "precision": 0.9009353741496599, + "recall": 0.673125794155019, + "f1-score": 0.7705454545454545, + "support": 3148.0 + }, + "accuracy": 0.9225529303467321, + "macro avg": { + "precision": 0.9135674503969271, + "recall": 0.8277015598903186, + "f1-score": 0.8619799993288365, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9213187931644278, + "recall": 0.9225529303467321, + "f1-score": 0.9180864131544908, + "support": 16295.0 + }, + "cross-entropy": 0.1796124403053473, + "roc-auc": 0.9623596978704976, + "score": 0.9623596978704976 + } + }, + "time": "0:06:50.247903", + "diversity": { + "train": { + "mean_centered_corr": 0.08324653655290604, + "mean_pairwise_disagreement": 0.03557734342644149, + "member_std": 0.03504224866628647 + }, + "val": { + "mean_centered_corr": 0.09014386683702469, + "mean_pairwise_disagreement": 0.03127462845078635, + "member_std": 0.035533007234334946 + }, + "test": { + "mean_centered_corr": 0.11173245310783386, + "mean_pairwise_disagreement": 0.03804637777271872, + "member_std": 0.03772429749369621 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_rla-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_rla-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_rla-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_rla-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..af02eb05d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_rla-greedy-heads-evaluation/0/report.json @@ -0,0 +1,675 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.901321291923523, + "min_features_per_member": 256.0, + "mean_features_per_member": 268.59375, + "max_features_per_member": 279.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.842281341552734, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8197222809700668, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_rla-evaluation/_esam_diagnostics/all_minus_rla-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9394039613420057, + "recall": 0.9833874628374851, + "f1-score": 0.9608926548211629, + "support": 181971.0 + }, + "1": { + "precision": 0.9106388010287032, + "recall": 0.7274315804387352, + "f1-score": 0.8087899393525689, + "support": 42349.0 + }, + "accuracy": 0.9350659771754636, + "macro avg": { + "precision": 0.9250213811853545, + "recall": 0.8554095216381101, + "f1-score": 0.884841297086866, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9339734345316096, + "recall": 0.9350659771754636, + "f1-score": 0.9321774359491074, + "support": 224320.0 + }, + "cross-entropy": 0.1522778155562675, + "roc-auc": 0.9742048378504812, + "score": 0.9742048378504812 + }, + "val": { + "0": { + "precision": 0.931451157538114, + "recall": 0.9853652708918225, + "f1-score": 0.95764999564599, + "support": 16741.0 + }, + "1": { + "precision": 0.8990939044481054, + "recall": 0.6426258463350015, + "f1-score": 0.7495278969957082, + "support": 3397.0 + }, + "accuracy": 0.927549905651008, + "macro avg": { + "precision": 0.9152725309931098, + "recall": 0.813995558613412, + "f1-score": 0.853588946320849, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9259929398031473, + "recall": 0.927549905651008, + "f1-score": 0.9225426975471218, + "support": 20138.0 + }, + "cross-entropy": 0.17448991850377593, + "roc-auc": 0.9594920636885601, + "score": 0.9594920636885601 + }, + "test": { + "0": { + "precision": 0.9245997712978845, + "recall": 0.984026774169012, + "f1-score": 0.9533881130476436, + "support": 13147.0 + }, + "1": { + "precision": 0.9088145896656535, + "recall": 0.6648665819567979, + "f1-score": 0.7679324894514767, + "support": 3148.0 + }, + "accuracy": 0.9223688247928813, + "macro avg": { + "precision": 0.916707180481769, + "recall": 0.824446678062905, + "f1-score": 0.8606603012495602, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9215502621368988, + "recall": 0.9223688247928813, + "f1-score": 0.9175602945093967, + "support": 16295.0 + }, + "cross-entropy": 0.1794172260622648, + "roc-auc": 0.9630834801355295, + "score": 0.9630834801355295 + } + }, + "time": "0:06:50.247903", + "diversity": { + "train": { + "mean_centered_corr": 0.08324653655290604, + "mean_pairwise_disagreement": 0.03557734342644149, + "member_std": 0.03504224866628647 + }, + "val": { + "mean_centered_corr": 0.09014386683702469, + "mean_pairwise_disagreement": 0.03127462845078635, + "member_std": 0.035533007234334946 + }, + "test": { + "mean_centered_corr": 0.11173245310783386, + "mean_pairwise_disagreement": 0.03804637777271872, + "member_std": 0.03772429749369621 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31, + 28, + 17, + 30, + 22, + 20, + 10, + 29, + 28 + ], + "head_selection_time": "0:00:15.790811" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.05-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.05-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.05-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.05-best-head-evaluation/0/report.json new file mode 100644 index 000000000..9894cb8d4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.05-best-head-evaluation/0/report.json @@ -0,0 +1,621 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9374104608021663, + "recall": 0.9816509223997231, + "f1-score": 0.9590207500067108, + "support": 181971.0 + }, + "1": { + "precision": 0.9010989010989011, + "recall": 0.7183640699898463, + "f1-score": 0.7994218893706477, + "support": 42349.0 + }, + "accuracy": 0.9319454350927246, + "macro avg": { + "precision": 0.9192546809505338, + "recall": 0.8500074961947847, + "f1-score": 0.8792213196886793, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9305552617923875, + "recall": 0.9319454350927246, + "f1-score": 0.9288903463464192, + "support": 224320.0 + }, + "cross-entropy": 0.15647591244364484, + "roc-auc": 0.9711251957175436, + "score": 0.9711251957175436 + }, + "val": { + "0": { + "precision": 0.9311434699175979, + "recall": 0.9854847380682158, + "f1-score": 0.9575437476421255, + "support": 16741.0 + }, + "1": { + "precision": 0.8995867768595042, + "recall": 0.6408595819841036, + "f1-score": 0.7484957882069796, + "support": 3397.0 + }, + "accuracy": 0.9273512761942596, + "macro avg": { + "precision": 0.915365123388551, + "recall": 0.8131721600261597, + "f1-score": 0.8530197679245526, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9258202955051267, + "recall": 0.9273512761942596, + "f1-score": 0.9222802697296619, + "support": 20138.0 + }, + "cross-entropy": 0.17757960670558423, + "roc-auc": 0.9577627701557911, + "score": 0.9577627701557911 + }, + "test": { + "0": { + "precision": 0.9243691471870755, + "recall": 0.9835703962881266, + "f1-score": 0.9530512971698113, + "support": 13147.0 + }, + "1": { + "precision": 0.9063313096270599, + "recall": 0.6639135959339263, + "f1-score": 0.7664099743307664, + "support": 3148.0 + }, + "accuracy": 0.9218165081313286, + "macro avg": { + "precision": 0.9153502284070677, + "recall": 0.8237419961110264, + "f1-score": 0.8597306357502889, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9208844517198199, + "recall": 0.9218165081313286, + "f1-score": 0.9169944156541737, + "support": 16295.0 + }, + "cross-entropy": 0.1822582690101261, + "roc-auc": 0.9622852537657216, + "score": 0.9622852537657216 + } + }, + "time": "0:03:33.494374", + "diversity": { + "train": { + "mean_centered_corr": 0.03106696531176567, + "mean_pairwise_disagreement": 0.027885827346302516, + "member_std": 0.025465095415711403 + }, + "val": { + "mean_centered_corr": 0.03693411499261856, + "mean_pairwise_disagreement": 0.02448668702084648, + "member_std": 0.02594522386789322 + }, + "test": { + "mean_centered_corr": 0.0499531626701355, + "mean_pairwise_disagreement": 0.0303318849043344, + "member_std": 0.027707822620868683 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 22 + ], + "head_selection_time": "0:00:01.036071" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.05-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.05-evaluation/0.toml new file mode 100644 index 000000000..d4cdc115c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.05-evaluation/0.toml @@ -0,0 +1,45 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 + +[cf_fisd] +lambda = 0.05 +variant = "raw" +dataset_name = "homesite-insurance" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.05-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.05-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.05-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.05-evaluation/0/report.json new file mode 100644 index 000000000..f4fc30df1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.05-evaluation/0/report.json @@ -0,0 +1,617 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9422552559823976, + "recall": 0.983695204180886, + "f1-score": 0.9625294062294156, + "support": 181971.0 + }, + "1": { + "precision": 0.9136143946893379, + "recall": 0.7409620061866868, + "f1-score": 0.8182802007953582, + "support": 42349.0 + }, + "accuracy": 0.9378700071326677, + "macro avg": { + "precision": 0.9279348253358677, + "recall": 0.8623286051837864, + "f1-score": 0.8904048035123869, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9368481953774591, + "recall": 0.9378700071326677, + "f1-score": 0.9352968384649413, + "support": 224320.0 + }, + "cross-entropy": 0.1435463636983052, + "roc-auc": 0.976569916089968, + "score": 0.976569916089968 + }, + "val": { + "0": { + "precision": 0.9331481061465184, + "recall": 0.9830356609521534, + "f1-score": 0.9574424760741194, + "support": 16741.0 + }, + "1": { + "precision": 0.8864908073541167, + "recall": 0.6529290550485722, + "f1-score": 0.7519918630276318, + "support": 3397.0 + }, + "accuracy": 0.9273512761942596, + "macro avg": { + "precision": 0.9098194567503175, + "recall": 0.8179823580003629, + "f1-score": 0.8547171695508755, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9252776699563413, + "recall": 0.9273512761942596, + "f1-score": 0.9227858203725146, + "support": 20138.0 + }, + "cross-entropy": 0.17365686034344033, + "roc-auc": 0.9587449454385457, + "score": 0.9587449454385457 + }, + "test": { + "0": { + "precision": 0.9279746470757707, + "recall": 0.9799954362211911, + "f1-score": 0.9532758684473382, + "support": 13147.0 + }, + "1": { + "precision": 0.8909166321028619, + "recall": 0.6823379923761118, + "f1-score": 0.7728008634646519, + "support": 3148.0 + }, + "accuracy": 0.9224915618287818, + "macro avg": { + "precision": 0.9094456395893162, + "recall": 0.8311667142986514, + "f1-score": 0.863038365955995, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9208154797769234, + "recall": 0.9224915618287818, + "f1-score": 0.9184102461284983, + "support": 16295.0 + }, + "cross-entropy": 0.17782881823892016, + "roc-auc": 0.9627875642149869, + "score": 0.9627875642149869 + } + }, + "time": "0:03:33.494374", + "diversity": { + "train": { + "mean_centered_corr": 0.03106696531176567, + "mean_pairwise_disagreement": 0.027885827346302516, + "member_std": 0.025465095415711403 + }, + "val": { + "mean_centered_corr": 0.03693411499261856, + "mean_pairwise_disagreement": 0.02448668702084648, + "member_std": 0.02594522386789322 + }, + "test": { + "mean_centered_corr": 0.0499531626701355, + "mean_pairwise_disagreement": 0.0303318849043344, + "member_std": 0.027707822620868683 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.05-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.05-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.05-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.05-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..960e36c0c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.05-greedy-heads-evaluation/0/report.json @@ -0,0 +1,634 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9419323202817703, + "recall": 0.9817223623544411, + "f1-score": 0.961415820616798, + "support": 181971.0 + }, + "1": { + "precision": 0.9040447752582078, + "recall": 0.739946633922879, + "f1-score": 0.8138058199477997, + "support": 42349.0 + }, + "accuracy": 0.9360779243937233, + "macro avg": { + "precision": 0.9229885477699891, + "recall": 0.86083449813866, + "f1-score": 0.8876108202822988, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9347795936225207, + "recall": 0.9360779243937233, + "f1-score": 0.9335487739052636, + "support": 224320.0 + }, + "cross-entropy": 0.14666618497849432, + "roc-auc": 0.9751858546067549, + "score": 0.9751858546067549 + }, + "val": { + "0": { + "precision": 0.9337947466954104, + "recall": 0.9832148617167433, + "f1-score": 0.9578677839851024, + "support": 16741.0 + }, + "1": { + "precision": 0.8880923934687376, + "recall": 0.656461583750368, + "f1-score": 0.7549085985104943, + "support": 3397.0 + }, + "accuracy": 0.9280961366570663, + "macro avg": { + "precision": 0.910943570082074, + "recall": 0.8198382227335557, + "f1-score": 0.8563881912477984, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9260853965160973, + "recall": 0.9280961366570663, + "f1-score": 0.9236313973996797, + "support": 20138.0 + }, + "cross-entropy": 0.17262425691887265, + "roc-auc": 0.9594017599375494, + "score": 0.9594017599375494 + }, + "test": { + "0": { + "precision": 0.9273996258454454, + "recall": 0.980375751121929, + "f1-score": 0.9531521538177112, + "support": 13147.0 + }, + "1": { + "precision": 0.8923654568210263, + "recall": 0.6794790343074968, + "f1-score": 0.7715058611361587, + "support": 3148.0 + }, + "accuracy": 0.9222460877569807, + "macro avg": { + "precision": 0.9098825413332359, + "recall": 0.8299273927147128, + "f1-score": 0.862329007476935, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9206314414889636, + "recall": 0.9222460877569807, + "f1-score": 0.9180602526602073, + "support": 16295.0 + }, + "cross-entropy": 0.17716065755132018, + "roc-auc": 0.9632589952205967, + "score": 0.9632589952205967 + } + }, + "time": "0:03:33.494374", + "diversity": { + "train": { + "mean_centered_corr": 0.03106696531176567, + "mean_pairwise_disagreement": 0.027885827346302516, + "member_std": 0.025465095415711403 + }, + "val": { + "mean_centered_corr": 0.03693411499261856, + "mean_pairwise_disagreement": 0.02448668702084648, + "member_std": 0.02594522386789322 + }, + "test": { + "mean_centered_corr": 0.0499531626701355, + "mean_pairwise_disagreement": 0.0303318849043344, + "member_std": 0.027707822620868683 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 22, + 31, + 15, + 28, + 26, + 29, + 24, + 2, + 15, + 31, + 9, + 26, + 20, + 28 + ], + "head_selection_time": "0:00:14.082869" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.1-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.1-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.1-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.1-best-head-evaluation/0/report.json new file mode 100644 index 000000000..42c30294b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.1-best-head-evaluation/0/report.json @@ -0,0 +1,621 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2420, + "metrics": { + "train": { + "0": { + "precision": 0.9511390416339356, + "recall": 0.9648020838485253, + "f1-score": 0.9579218455024607, + "support": 181971.0 + }, + "1": { + "precision": 0.8388070970177426, + "recall": 0.7870315709934119, + "f1-score": 0.8120949271477998, + "support": 42349.0 + }, + "accuracy": 0.9312410841654779, + "macro avg": { + "precision": 0.8949730693258391, + "recall": 0.8759168274209685, + "f1-score": 0.8850083863251303, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9299320804956013, + "recall": 0.9312410841654779, + "f1-score": 0.9303914239377251, + "support": 224320.0 + }, + "cross-entropy": 0.15615645016500299, + "roc-auc": 0.9724991426058968, + "score": 0.9724991426058968 + }, + "val": { + "0": { + "precision": 0.9410138248847926, + "recall": 0.9758078967803596, + "f1-score": 0.958095070525791, + "support": 16741.0 + }, + "1": { + "precision": 0.8542116630669546, + "recall": 0.6985575507801001, + "f1-score": 0.768582995951417, + "support": 3397.0 + }, + "accuracy": 0.9290396265766213, + "macro avg": { + "precision": 0.8976127439758737, + "recall": 0.8371827237802298, + "f1-score": 0.863339033238604, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9263715096749804, + "recall": 0.9290396265766213, + "f1-score": 0.9261270241791256, + "support": 20138.0 + }, + "cross-entropy": 0.17528768630111824, + "roc-auc": 0.957813544584969, + "score": 0.957813544584969 + }, + "test": { + "0": { + "precision": 0.9342733348020856, + "recall": 0.9676732334372861, + "f1-score": 0.9506800179345389, + "support": 13147.0 + }, + "1": { + "precision": 0.8412994772218073, + "recall": 0.71569250317662, + "f1-score": 0.7734294541709578, + "support": 3148.0 + }, + "accuracy": 0.9189935563056152, + "macro avg": { + "precision": 0.8877864060119465, + "recall": 0.841682868306953, + "f1-score": 0.8620547360527484, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9163118924171383, + "recall": 0.9189935563056152, + "f1-score": 0.9164373192706695, + "support": 16295.0 + }, + "cross-entropy": 0.1803789882797406, + "roc-auc": 0.961490313471295, + "score": 0.961490313471295 + } + }, + "time": "0:03:47.889216", + "diversity": { + "train": { + "mean_centered_corr": 0.0506281740963459, + "mean_pairwise_disagreement": 0.03327413710540242, + "member_std": 0.03079444356262684 + }, + "val": { + "mean_centered_corr": 0.056434109807014465, + "mean_pairwise_disagreement": 0.030276074921749604, + "member_std": 0.03229342773556709 + }, + "test": { + "mean_centered_corr": 0.06734751909971237, + "mean_pairwise_disagreement": 0.037917825574834946, + "member_std": 0.03452758118510246 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25 + ], + "head_selection_time": "0:00:00.859514" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.1-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.1-evaluation/0.toml new file mode 100644 index 000000000..e655e5952 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.1-evaluation/0.toml @@ -0,0 +1,45 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "homesite-insurance" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.1-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.1-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.1-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.1-evaluation/0/report.json new file mode 100644 index 000000000..c86e0c52b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.1-evaluation/0/report.json @@ -0,0 +1,617 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2420, + "metrics": { + "train": { + "0": { + "precision": 0.9480090078156047, + "recall": 0.9831951244978595, + "f1-score": 0.965281524483674, + "support": 181971.0 + }, + "1": { + "precision": 0.9140890574518893, + "recall": 0.7683062173841176, + "f1-score": 0.8348814533511238, + "support": 42349.0 + }, + "accuracy": 0.942626604850214, + "macro avg": { + "precision": 0.931049032633747, + "recall": 0.8757506709409886, + "f1-score": 0.9000814889173989, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9416053167583962, + "recall": 0.942626604850214, + "f1-score": 0.9406635117679449, + "support": 224320.0 + }, + "cross-entropy": 0.13436786219631347, + "roc-auc": 0.9804692077039372, + "score": 0.9804692077039372 + }, + "val": { + "0": { + "precision": 0.9353075170842825, + "recall": 0.9810644525416642, + "f1-score": 0.957639718958631, + "support": 16741.0 + }, + "1": { + "precision": 0.8770364623739333, + "recall": 0.6655872828966736, + "f1-score": 0.7568200836820084, + "support": 3397.0 + }, + "accuracy": 0.9278478498361307, + "macro avg": { + "precision": 0.9061719897291078, + "recall": 0.8233258677191688, + "f1-score": 0.8572299013203197, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9254780020951546, + "recall": 0.9278478498361307, + "f1-score": 0.9237642446794232, + "support": 20138.0 + }, + "cross-entropy": 0.1724175473381701, + "roc-auc": 0.9589599740471012, + "score": 0.9589599740471012 + }, + "test": { + "0": { + "precision": 0.931104503589818, + "recall": 0.9765726021145509, + "f1-score": 0.9532967032967034, + "support": 13147.0 + }, + "1": { + "precision": 0.8770949720670391, + "recall": 0.6982210927573063, + "f1-score": 0.7775026529890343, + "support": 3148.0 + }, + "accuracy": 0.9227984044185333, + "macro avg": { + "precision": 0.9040997378284286, + "recall": 0.8373968474359286, + "f1-score": 0.8653996781428688, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9206705051096887, + "recall": 0.9227984044185333, + "f1-score": 0.919335385692006, + "support": 16295.0 + }, + "cross-entropy": 0.17668487055650292, + "roc-auc": 0.9628908822909436, + "score": 0.9628908822909436 + } + }, + "time": "0:03:47.889216", + "diversity": { + "train": { + "mean_centered_corr": 0.0506281740963459, + "mean_pairwise_disagreement": 0.03327413710540242, + "member_std": 0.03079444356262684 + }, + "val": { + "mean_centered_corr": 0.056434109807014465, + "mean_pairwise_disagreement": 0.030276074921749604, + "member_std": 0.03229342773556709 + }, + "test": { + "mean_centered_corr": 0.06734751909971237, + "mean_pairwise_disagreement": 0.037917825574834946, + "member_std": 0.03452758118510246 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.1-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.1-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.1-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.1-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..c890b44c4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.1-greedy-heads-evaluation/0/report.json @@ -0,0 +1,639 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2420, + "metrics": { + "train": { + "0": { + "precision": 0.9470743368373016, + "recall": 0.9808101290865028, + "f1-score": 0.9636470639051465, + "support": 181971.0 + }, + "1": { + "precision": 0.9026403100342933, + "recall": 0.7644808614134927, + "f1-score": 0.8278357369336197, + "support": 42349.0 + }, + "accuracy": 0.9399696861626248, + "macro avg": { + "precision": 0.9248573234357975, + "recall": 0.8726454952499978, + "f1-score": 0.8957414004193831, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9386857107625842, + "recall": 0.9399696861626248, + "f1-score": 0.9380074691926055, + "support": 224320.0 + }, + "cross-entropy": 0.13890143673129085, + "roc-auc": 0.9784781707405066, + "score": 0.9784781707405066 + }, + "val": { + "0": { + "precision": 0.9363594890510949, + "recall": 0.9808255181888776, + "f1-score": 0.9580768445313184, + "support": 16741.0 + }, + "1": { + "precision": 0.8766333589546502, + "recall": 0.6714748307329997, + "f1-score": 0.7604600766794466, + "support": 3397.0 + }, + "accuracy": 0.9286423676631245, + "macro avg": { + "precision": 0.9064964240028726, + "recall": 0.8261501744609387, + "f1-score": 0.8592684606053824, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9262845231191442, + "recall": 0.9286423676631245, + "f1-score": 0.9247416493583713, + "support": 20138.0 + }, + "cross-entropy": 0.17179269209939912, + "roc-auc": 0.9594844321379927, + "score": 0.9594844321379927 + }, + "test": { + "0": { + "precision": 0.9321799811005307, + "recall": 0.9754316574123374, + "f1-score": 0.9533154921201308, + "support": 13147.0 + }, + "1": { + "precision": 0.8727344365642238, + "recall": 0.7036213468869124, + "f1-score": 0.7791065775589167, + "support": 3148.0 + }, + "accuracy": 0.9229211414544338, + "macro avg": { + "precision": 0.9024572088323772, + "recall": 0.8395265021496249, + "f1-score": 0.8662110348395238, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9206958096245998, + "recall": 0.9229211414544338, + "f1-score": 0.9196604038698268, + "support": 16295.0 + }, + "cross-entropy": 0.1758390088286691, + "roc-auc": 0.9634382071404677, + "score": 0.9634382071404677 + } + }, + "time": "0:03:47.889216", + "diversity": { + "train": { + "mean_centered_corr": 0.0506281740963459, + "mean_pairwise_disagreement": 0.03327413710540242, + "member_std": 0.03079444356262684 + }, + "val": { + "mean_centered_corr": 0.056434109807014465, + "mean_pairwise_disagreement": 0.030276074921749604, + "member_std": 0.03229342773556709 + }, + "test": { + "mean_centered_corr": 0.06734751909971237, + "mean_pairwise_disagreement": 0.037917825574834946, + "member_std": 0.03452758118510246 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25, + 2, + 0, + 28, + 7, + 26, + 18, + 29, + 24, + 15, + 4, + 7, + 27, + 19, + 31, + 0, + 2, + 28, + 11 + ], + "head_selection_time": "0:00:17.073344" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.2-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.2-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.2-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.2-best-head-evaluation/0/report.json new file mode 100644 index 000000000..c778139f5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.2-best-head-evaluation/0/report.json @@ -0,0 +1,621 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.2, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.2, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2200, + "metrics": { + "train": { + "0": { + "precision": 0.9425672803027082, + "recall": 0.9746662929807497, + "f1-score": 0.9583480800456045, + "support": 181971.0 + }, + "1": { + "precision": 0.8724828501880947, + "recall": 0.7448109754657725, + "f1-score": 0.8036075973554477, + "support": 42349.0 + }, + "accuracy": 0.9312722895863053, + "macro avg": { + "precision": 0.9075250652454014, + "recall": 0.8597386342232611, + "f1-score": 0.880977838700526, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9293361572154945, + "recall": 0.9312722895863053, + "f1-score": 0.9291348814835261, + "support": 224320.0 + }, + "cross-entropy": 0.15513930110205976, + "roc-auc": 0.9715887850135724, + "score": 0.9715887850135724 + }, + "val": { + "0": { + "precision": 0.9368529428587771, + "recall": 0.9783764410728153, + "f1-score": 0.9571645628798504, + "support": 16741.0 + }, + "1": { + "precision": 0.863653483992467, + "recall": 0.6750073594347954, + "f1-score": 0.7577660277594184, + "support": 3397.0 + }, + "accuracy": 0.9272023041016982, + "macro avg": { + "precision": 0.900253213425622, + "recall": 0.8266919002538053, + "f1-score": 0.8574652953196344, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9245052140987783, + "recall": 0.9272023041016982, + "f1-score": 0.9235288083955865, + "support": 20138.0 + }, + "cross-entropy": 0.17668127284059484, + "roc-auc": 0.9576897428988642, + "score": 0.9576897428988642 + }, + "test": { + "0": { + "precision": 0.9302224476487211, + "recall": 0.9764965391344033, + "f1-score": 0.9527979812973133, + "support": 13147.0 + }, + "1": { + "precision": 0.876102646351243, + "recall": 0.6940914866581956, + "f1-score": 0.7745480326125488, + "support": 3148.0 + }, + "accuracy": 0.9219392451672292, + "macro avg": { + "precision": 0.903162546999982, + "recall": 0.8352940128962995, + "f1-score": 0.863673006954931, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9197671463609358, + "recall": 0.9219392451672292, + "f1-score": 0.9183622133648409, + "support": 16295.0 + }, + "cross-entropy": 0.18215599030843227, + "roc-auc": 0.9612223171103336, + "score": 0.9612223171103336 + } + }, + "time": "0:03:40.803387", + "diversity": { + "train": { + "mean_centered_corr": 0.03675549104809761, + "mean_pairwise_disagreement": 0.03015326247641618, + "member_std": 0.027854571118950844 + }, + "val": { + "mean_centered_corr": 0.03806319460272789, + "mean_pairwise_disagreement": 0.026045187400485042, + "member_std": 0.02860930748283863 + }, + "test": { + "mean_centered_corr": 0.04883061721920967, + "mean_pairwise_disagreement": 0.032468771342881744, + "member_std": 0.030201883986592293 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 28 + ], + "head_selection_time": "0:00:01.169734" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.2-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.2-evaluation/0.toml new file mode 100644 index 000000000..2010232d3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.2-evaluation/0.toml @@ -0,0 +1,45 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 + +[cf_fisd] +lambda = 0.2 +variant = "raw" +dataset_name = "homesite-insurance" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.2-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.2-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.2-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.2-evaluation/0/report.json new file mode 100644 index 000000000..882db6358 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.2-evaluation/0/report.json @@ -0,0 +1,617 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.2, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.2, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2200, + "metrics": { + "train": { + "0": { + "precision": 0.942135079458505, + "recall": 0.9851954432299652, + "f1-score": 0.9631842346323393, + "support": 181971.0 + }, + "1": { + "precision": 0.9208392101551481, + "recall": 0.7399938605398002, + "f1-score": 0.8205705607415457, + "support": 42349.0 + }, + "accuracy": 0.9389042439372325, + "macro avg": { + "precision": 0.9314871448068266, + "recall": 0.8625946518848827, + "f1-score": 0.8918773976869425, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9381146676845755, + "recall": 0.9389042439372325, + "f1-score": 0.9362604361498089, + "support": 224320.0 + }, + "cross-entropy": 0.14122963803899807, + "roc-auc": 0.9777868186004166, + "score": 0.9777868186004166 + }, + "val": { + "0": { + "precision": 0.9316268294061141, + "recall": 0.9848276685980527, + "f1-score": 0.9574888204889948, + "support": 16741.0 + }, + "1": { + "precision": 0.8959442851290454, + "recall": 0.6438033559022667, + "f1-score": 0.7492291880781089, + "support": 3397.0 + }, + "accuracy": 0.9273016188300724, + "macro avg": { + "precision": 0.9137855572675797, + "recall": 0.8143155122501597, + "f1-score": 0.8533590042835518, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9256076813820202, + "recall": 0.9273016188300724, + "f1-score": 0.9223583223610883, + "support": 20138.0 + }, + "cross-entropy": 0.1736664220536342, + "roc-auc": 0.9587151577734279, + "score": 0.9587151577734279 + }, + "test": { + "0": { + "precision": 0.9260772926077293, + "recall": 0.9824294515859131, + "f1-score": 0.9534214217169853, + "support": 13147.0 + }, + "1": { + "precision": 0.901618398637138, + "recall": 0.6724904701397713, + "f1-score": 0.7703784570596798, + "support": 3148.0 + }, + "accuracy": 0.9225529303467321, + "macro avg": { + "precision": 0.9138478456224337, + "recall": 0.8274599608628421, + "f1-score": 0.8618999393883325, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9213521254877893, + "recall": 0.9225529303467321, + "f1-score": 0.9180597001618336, + "support": 16295.0 + }, + "cross-entropy": 0.17768434161277474, + "roc-auc": 0.9626366101271623, + "score": 0.9626366101271623 + } + }, + "time": "0:03:40.803387", + "diversity": { + "train": { + "mean_centered_corr": 0.03675549104809761, + "mean_pairwise_disagreement": 0.03015326247641618, + "member_std": 0.027854571118950844 + }, + "val": { + "mean_centered_corr": 0.03806319460272789, + "mean_pairwise_disagreement": 0.026045187400485042, + "member_std": 0.02860930748283863 + }, + "test": { + "mean_centered_corr": 0.04883061721920967, + "mean_pairwise_disagreement": 0.032468771342881744, + "member_std": 0.030201883986592293 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.2-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.2-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.2-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.2-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..7186c0353 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.2-greedy-heads-evaluation/0/report.json @@ -0,0 +1,634 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.2, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.2, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2200, + "metrics": { + "train": { + "0": { + "precision": 0.9428691849744482, + "recall": 0.9834863796978639, + "f1-score": 0.9627495756910655, + "support": 181971.0 + }, + "1": { + "precision": 0.9129237902057374, + "recall": 0.7439372830527285, + "f1-score": 0.8198129041491562, + "support": 42349.0 + }, + "accuracy": 0.9382623038516406, + "macro avg": { + "precision": 0.9278964875900928, + "recall": 0.8637118313752963, + "f1-score": 0.8912812399201109, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9372158436626608, + "recall": 0.9382623038516406, + "f1-score": 0.9357647990187744, + "support": 224320.0 + }, + "cross-entropy": 0.14301129796720108, + "roc-auc": 0.9769173245111457, + "score": 0.9769173245111457 + }, + "val": { + "0": { + "precision": 0.9320844416775143, + "recall": 0.9837524640105131, + "f1-score": 0.9572217378668991, + "support": 16741.0 + }, + "1": { + "precision": 0.8898339408667477, + "recall": 0.6467471298204298, + "f1-score": 0.7490623934538015, + "support": 3397.0 + }, + "accuracy": 0.9269043599165756, + "macro avg": { + "precision": 0.910959191272131, + "recall": 0.8152497969154715, + "f1-score": 0.8531420656603503, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9249573709031487, + "recall": 0.9269043599165756, + "f1-score": 0.922108156926821, + "support": 20138.0 + }, + "cross-entropy": 0.17259799258113656, + "roc-auc": 0.9594008895187635, + "score": 0.9594008895187635 + }, + "test": { + "0": { + "precision": 0.9257134662268751, + "recall": 0.9819730737050277, + "f1-score": 0.9530136935739859, + "support": 13147.0 + }, + "1": { + "precision": 0.8991060025542784, + "recall": 0.6709021601016518, + "f1-score": 0.768419137711479, + "support": 3148.0 + }, + "accuracy": 0.9218778766492789, + "macro avg": { + "precision": 0.9124097343905768, + "recall": 0.8264376169033398, + "f1-score": 0.8607164156427325, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9205732210202882, + "recall": 0.9218778766492789, + "f1-score": 0.9173522230704466, + "support": 16295.0 + }, + "cross-entropy": 0.1777880940764334, + "roc-auc": 0.9626779880984148, + "score": 0.9626779880984148 + } + }, + "time": "0:03:40.803387", + "diversity": { + "train": { + "mean_centered_corr": 0.03675549104809761, + "mean_pairwise_disagreement": 0.03015326247641618, + "member_std": 0.027854571118950844 + }, + "val": { + "mean_centered_corr": 0.03806319460272789, + "mean_pairwise_disagreement": 0.026045187400485042, + "member_std": 0.02860930748283863 + }, + "test": { + "mean_centered_corr": 0.04883061721920967, + "mean_pairwise_disagreement": 0.032468771342881744, + "member_std": 0.030201883986592293 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 28, + 24, + 4, + 26, + 14, + 5, + 25, + 21, + 16, + 4, + 24, + 15, + 26, + 3 + ], + "head_selection_time": "0:00:14.459347" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_cf_fisd-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_cf_fisd-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_cf_fisd-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_cf_fisd-best-head-evaluation/0/report.json new file mode 100644 index 000000000..1bc2f01c1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_cf_fisd-best-head-evaluation/0/report.json @@ -0,0 +1,631 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_cf_fisd-evaluation/_esam_diagnostics/esam_cf_fisd-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2200, + "metrics": { + "train": { + "0": { + "precision": 0.9395059644977903, + "recall": 0.9790076440751548, + "f1-score": 0.9588501398034946, + "support": 181971.0 + }, + "1": { + "precision": 0.8899071992622053, + "recall": 0.7291317386479019, + "f1-score": 0.801536724337093, + "support": 42349.0 + }, + "accuracy": 0.9318339871611983, + "macro avg": { + "precision": 0.9147065818799978, + "recall": 0.8540696913615284, + "f1-score": 0.8801934320702938, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9301422960377254, + "recall": 0.9318339871611983, + "f1-score": 0.9291512015385756, + "support": 224320.0 + }, + "cross-entropy": 0.15542274217664598, + "roc-auc": 0.9714271452985398, + "score": 0.9714271452985398 + }, + "val": { + "0": { + "precision": 0.9343040782663102, + "recall": 0.9811839197180575, + "f1-score": 0.9571703280694598, + "support": 16741.0 + }, + "1": { + "precision": 0.8768087602659367, + "recall": 0.6599941124521637, + "f1-score": 0.7531071548538797, + "support": 3397.0 + }, + "accuracy": 0.9270036746449498, + "macro avg": { + "precision": 0.9055564192661234, + "recall": 0.8205890160851106, + "f1-score": 0.8551387414616698, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9246054192511514, + "recall": 0.9270036746449498, + "f1-score": 0.9227477141349417, + "support": 20138.0 + }, + "cross-entropy": 0.17629531770563991, + "roc-auc": 0.9573917959108148, + "score": 0.9573917959108148 + }, + "test": { + "0": { + "precision": 0.9284371837501807, + "recall": 0.9769529170152886, + "f1-score": 0.9520773877914088, + "support": 13147.0 + }, + "1": { + "precision": 0.8768793173506705, + "recall": 0.6855146124523507, + "f1-score": 0.7694776252451417, + "support": 3148.0 + }, + "accuracy": 0.9206505062902731, + "macro avg": { + "precision": 0.9026582505504256, + "recall": 0.8312337647338197, + "f1-score": 0.8607775065182752, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9184768177835861, + "recall": 0.9206505062902731, + "f1-score": 0.9168012876075704, + "support": 16295.0 + }, + "cross-entropy": 0.18263783097893574, + "roc-auc": 0.9606087512633268, + "score": 0.9606087512633268 + } + }, + "time": "0:06:46.725623", + "diversity": { + "train": { + "mean_centered_corr": 0.030530203133821487, + "mean_pairwise_disagreement": 0.02995129905147025, + "member_std": 0.027598347514867783 + }, + "val": { + "mean_centered_corr": 0.03432288020849228, + "mean_pairwise_disagreement": 0.02602145999058112, + "member_std": 0.02860495075583458 + }, + "test": { + "mean_centered_corr": 0.04354812577366829, + "mean_pairwise_disagreement": 0.03329724633521068, + "member_std": 0.030326420441269875 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 7 + ], + "head_selection_time": "0:00:01.228380" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_cf_fisd-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_cf_fisd-evaluation/0.toml new file mode 100644 index 000000000..0963236a7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_cf_fisd-evaluation/0.toml @@ -0,0 +1,55 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "homesite-insurance" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_cf_fisd-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_cf_fisd-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_cf_fisd-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_cf_fisd-evaluation/0/report.json new file mode 100644 index 000000000..b4ca549ac --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_cf_fisd-evaluation/0/report.json @@ -0,0 +1,627 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_cf_fisd-evaluation/_esam_diagnostics/esam_cf_fisd-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2200, + "metrics": { + "train": { + "0": { + "precision": 0.9432183133443545, + "recall": 0.9845140159695775, + "f1-score": 0.9634238483069784, + "support": 181971.0 + }, + "1": { + "precision": 0.9180385085219009, + "recall": 0.7453304682519067, + "f1-score": 0.8227183276641774, + "support": 42349.0 + }, + "accuracy": 0.9393589514978602, + "macro avg": { + "precision": 0.9306284109331278, + "recall": 0.8649222421107421, + "f1-score": 0.8930710879855779, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9384646598385321, + "recall": 0.9393589514978602, + "f1-score": 0.9368602869049545, + "support": 224320.0 + }, + "cross-entropy": 0.1409082641420337, + "roc-auc": 0.9779900500547988, + "score": 0.9779900500547988 + }, + "val": { + "0": { + "precision": 0.9328840054372451, + "recall": 0.9838719311869064, + "f1-score": 0.9576997994011106, + "support": 16741.0 + }, + "1": { + "precision": 0.8912167606768735, + "recall": 0.6511627906976745, + "f1-score": 0.7525089300901514, + "support": 3397.0 + }, + "accuracy": 0.9277485351077565, + "macro avg": { + "precision": 0.9120503830570593, + "recall": 0.8175173609422904, + "f1-score": 0.855104364745631, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.925855321831575, + "recall": 0.9277485351077565, + "f1-score": 0.9230869588484575, + "support": 20138.0 + }, + "cross-entropy": 0.1728382451055005, + "roc-auc": 0.9589333075806602, + "score": 0.9589333075806602 + }, + "test": { + "0": { + "precision": 0.9278877293990644, + "recall": 0.9806800030425192, + "f1-score": 0.953553731232897, + "support": 13147.0 + }, + "1": { + "precision": 0.8941666666666667, + "recall": 0.681702668360864, + "f1-score": 0.7736121124729632, + "support": 3148.0 + }, + "accuracy": 0.9229211414544338, + "macro avg": { + "precision": 0.9110271980328655, + "recall": 0.8311913357016916, + "f1-score": 0.8635829218529301, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.921373221545024, + "recall": 0.9229211414544338, + "f1-score": 0.9187911527820672, + "support": 16295.0 + }, + "cross-entropy": 0.17708071781719004, + "roc-auc": 0.9627584002959787, + "score": 0.9627584002959787 + } + }, + "time": "0:06:46.725623", + "diversity": { + "train": { + "mean_centered_corr": 0.030530203133821487, + "mean_pairwise_disagreement": 0.02995129905147025, + "member_std": 0.027598347514867783 + }, + "val": { + "mean_centered_corr": 0.03432288020849228, + "mean_pairwise_disagreement": 0.02602145999058112, + "member_std": 0.02860495075583458 + }, + "test": { + "mean_centered_corr": 0.04354812577366829, + "mean_pairwise_disagreement": 0.03329724633521068, + "member_std": 0.030326420441269875 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_cf_fisd-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_cf_fisd-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_cf_fisd-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_cf_fisd-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..ed9567004 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_cf_fisd-greedy-heads-evaluation/0/report.json @@ -0,0 +1,642 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_cf_fisd-evaluation/_esam_diagnostics/esam_cf_fisd-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2200, + "metrics": { + "train": { + "0": { + "precision": 0.9421593289198092, + "recall": 0.9838435794714543, + "f1-score": 0.9625503705714893, + "support": 181971.0 + }, + "1": { + "precision": 0.9142807160767392, + "recall": 0.7404661267090132, + "f1-score": 0.8182446801570837, + "support": 42349.0 + }, + "accuracy": 0.9378967546362339, + "macro avg": { + "precision": 0.9282200224982742, + "recall": 0.8621548530902338, + "f1-score": 0.8903975253642865, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9368961719329549, + "recall": 0.9378967546362339, + "f1-score": 0.935307139101448, + "support": 224320.0 + }, + "cross-entropy": 0.14299189252771094, + "roc-auc": 0.9769227543613922, + "score": 0.9769227543613922 + }, + "val": { + "0": { + "precision": 0.9330086641372671, + "recall": 0.9841705991278896, + "f1-score": 0.957906976744186, + "support": 16741.0 + }, + "1": { + "precision": 0.8931020572811618, + "recall": 0.651751545481307, + "f1-score": 0.7535738597685501, + "support": 3397.0 + }, + "accuracy": 0.9280961366570663, + "macro avg": { + "precision": 0.9130553607092144, + "recall": 0.8179610723045982, + "f1-score": 0.855740418256368, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9262769756135711, + "recall": 0.9280961366570663, + "f1-score": 0.9234388270587042, + "support": 20138.0 + }, + "cross-entropy": 0.1721808588629276, + "roc-auc": 0.9595481045909983, + "score": 0.9595481045909983 + }, + "test": { + "0": { + "precision": 0.9277021674947793, + "recall": 0.9799193732410436, + "f1-score": 0.9530961012058888, + "support": 13147.0 + }, + "1": { + "precision": 0.8903654485049833, + "recall": 0.6810673443456162, + "f1-score": 0.7717782577393808, + "support": 3148.0 + }, + "accuracy": 0.9221847192390303, + "macro avg": { + "precision": 0.9090338079998813, + "recall": 0.8304933587933299, + "f1-score": 0.8624371794726349, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9204891578979779, + "recall": 0.9221847192390303, + "f1-score": 0.9180676525263818, + "support": 16295.0 + }, + "cross-entropy": 0.17638530532168833, + "roc-auc": 0.9631976229303886, + "score": 0.9631976229303886 + } + }, + "time": "0:06:46.725623", + "diversity": { + "train": { + "mean_centered_corr": 0.030530203133821487, + "mean_pairwise_disagreement": 0.02995129905147025, + "member_std": 0.027598347514867783 + }, + "val": { + "mean_centered_corr": 0.03432288020849228, + "mean_pairwise_disagreement": 0.02602145999058112, + "member_std": 0.02860495075583458 + }, + "test": { + "mean_centered_corr": 0.04354812577366829, + "mean_pairwise_disagreement": 0.03329724633521068, + "member_std": 0.030326420441269875 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 7, + 5, + 4, + 18, + 26, + 16, + 30, + 11, + 25, + 5, + 4, + 26 + ], + "head_selection_time": "0:00:13.901959" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_mfb-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_mfb-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_mfb-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_mfb-best-head-evaluation/0/report.json new file mode 100644 index 000000000..66229ec2b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_mfb-best-head-evaluation/0/report.json @@ -0,0 +1,308 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.901321291923523, + "min_features_per_member": 256.0, + "mean_features_per_member": 268.59375, + "max_features_per_member": 279.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.842281341552734, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8197222809700668, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_mfb-evaluation/_esam_diagnostics/esam_mfb-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9507099696263939, + "recall": 0.973567216754318, + "f1-score": 0.9620028399449387, + "support": 181971.0 + }, + "1": { + "precision": 0.873334386685627, + "recall": 0.7831117617889443, + "f1-score": 0.8257659698965427, + "support": 42349.0 + }, + "accuracy": 0.9376114479315264, + "macro avg": { + "precision": 0.9120221781560105, + "recall": 0.8783394892716312, + "f1-score": 0.8938844049207407, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9361023619143818, + "recall": 0.9376114479315264, + "f1-score": 0.9362829076621306, + "support": 224320.0 + }, + "cross-entropy": 0.14378673331190528, + "roc-auc": 0.9763699988763426, + "score": 0.9763699988763426 + }, + "val": { + "0": { + "precision": 0.9398387190346348, + "recall": 0.9676841287856162, + "f1-score": 0.9535581847077521, + "support": 16741.0 + }, + "1": { + "precision": 0.8135125818683213, + "recall": 0.694730644686488, + "f1-score": 0.7494442680215941, + "support": 3397.0 + }, + "accuracy": 0.9216406793127421, + "macro avg": { + "precision": 0.8766756504514781, + "recall": 0.8312073867360521, + "f1-score": 0.8515012263646731, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9185292599049314, + "recall": 0.9216406793127421, + "f1-score": 0.9191270110567998, + "support": 20138.0 + }, + "cross-entropy": 0.18378915635489101, + "roc-auc": 0.9551748656394307, + "score": 0.9551748656394307 + }, + "test": { + "0": { + "precision": 0.9405636336381352, + "recall": 0.9545143378717578, + "f1-score": 0.9474876363773642, + "support": 13147.0 + }, + "1": { + "precision": 0.7974940738232306, + "recall": 0.7480940279542567, + "f1-score": 0.7720045894115719, + "support": 3148.0 + }, + "accuracy": 0.9146363915311445, + "macro avg": { + "precision": 0.869028853730683, + "recall": 0.8513041829130072, + "f1-score": 0.8597461128944681, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9129242979954644, + "recall": 0.9146363915311445, + "f1-score": 0.9135864009156696, + "support": 16295.0 + }, + "cross-entropy": 0.1875048907820305, + "roc-auc": 0.9597148541915197, + "score": 0.9597148541915197 + } + }, + "time": "0:05:30.729154", + "diversity": { + "train": { + "mean_centered_corr": 0.08804295212030411, + "mean_pairwise_disagreement": 0.03950866022329851, + "member_std": 0.03889348357915878 + }, + "val": { + "mean_centered_corr": 0.0962420254945755, + "mean_pairwise_disagreement": 0.03718395490470592, + "member_std": 0.042661890387535095 + }, + "test": { + "mean_centered_corr": 0.11283686012029648, + "mean_pairwise_disagreement": 0.04520397608607429, + "member_std": 0.04494225233793259 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31 + ], + "head_selection_time": "0:00:01.037395" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_mfb-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_mfb-evaluation/0.toml new file mode 100644 index 000000000..e5a6651c3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_mfb-evaluation/0.toml @@ -0,0 +1,57 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_mfb-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_mfb-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_mfb-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_mfb-evaluation/0/report.json new file mode 100644 index 000000000..31058d82c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_mfb-evaluation/0/report.json @@ -0,0 +1,304 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.901321291923523, + "min_features_per_member": 256.0, + "mean_features_per_member": 268.59375, + "max_features_per_member": 279.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.842281341552734, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8197222809700668, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_mfb-evaluation/_esam_diagnostics/esam_mfb-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9469197964827833, + "recall": 0.9859318243016745, + "f1-score": 0.966032107560057, + "support": 181971.0 + }, + "1": { + "precision": 0.9265465396533915, + "recall": 0.7625209568112589, + "f1-score": 0.8365694744886725, + "support": 42349.0 + }, + "accuracy": 0.9437544579172611, + "macro avg": { + "precision": 0.9367331680680875, + "recall": 0.8742263905564667, + "f1-score": 0.9013007910243647, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9430735631845134, + "recall": 0.9437544579172611, + "f1-score": 0.9415910722179561, + "support": 224320.0 + }, + "cross-entropy": 0.13682316815786993, + "roc-auc": 0.9807455131029597, + "score": 0.9807455131029597 + }, + "val": { + "0": { + "precision": 0.9328421649192405, + "recall": 0.9832148617167433, + "f1-score": 0.9573663700343162, + "support": 16741.0 + }, + "1": { + "precision": 0.887284396309667, + "recall": 0.6511627906976745, + "f1-score": 0.7511035653650254, + "support": 3397.0 + }, + "accuracy": 0.9272023041016982, + "macro avg": { + "precision": 0.9100632806144537, + "recall": 0.8171888262072089, + "f1-score": 0.8542349676996708, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9251572041502107, + "recall": 0.9272023041016982, + "f1-score": 0.922572708922906, + "support": 20138.0 + }, + "cross-entropy": 0.17538838012996275, + "roc-auc": 0.9580893002900325, + "score": 0.9580893002900325 + }, + "test": { + "0": { + "precision": 0.9285301344124873, + "recall": 0.9773332319160265, + "f1-score": 0.9523068371317398, + "support": 13147.0 + }, + "1": { + "precision": 0.8787138787138787, + "recall": 0.6858322744599746, + "f1-score": 0.7703835860838537, + "support": 3148.0 + }, + "accuracy": 0.9210187173979748, + "macro avg": { + "precision": 0.903622006563183, + "recall": 0.8315827531880006, + "f1-score": 0.8613452116077968, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9189062268985738, + "recall": 0.9210187173979748, + "f1-score": 0.9171614309151859, + "support": 16295.0 + }, + "cross-entropy": 0.17996585840914717, + "roc-auc": 0.9618081204528327, + "score": 0.9618081204528327 + } + }, + "time": "0:05:30.729154", + "diversity": { + "train": { + "mean_centered_corr": 0.08804295212030411, + "mean_pairwise_disagreement": 0.03950866022329851, + "member_std": 0.03889348357915878 + }, + "val": { + "mean_centered_corr": 0.0962420254945755, + "mean_pairwise_disagreement": 0.03718395490470592, + "member_std": 0.042661890387535095 + }, + "test": { + "mean_centered_corr": 0.11283686012029648, + "mean_pairwise_disagreement": 0.04520397608607429, + "member_std": 0.04494225233793259 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_mfb-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_mfb-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_mfb-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_mfb-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..58a2a2d41 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_mfb-greedy-heads-evaluation/0/report.json @@ -0,0 +1,317 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.901321291923523, + "min_features_per_member": 256.0, + "mean_features_per_member": 268.59375, + "max_features_per_member": 279.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.842281341552734, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8197222809700668, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_mfb-evaluation/_esam_diagnostics/esam_mfb-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9477218327013917, + "recall": 0.9834753889355996, + "f1-score": 0.9652676459721634, + "support": 181971.0 + }, + "1": { + "precision": 0.9152575808815241, + "recall": 0.7668894188764788, + "f1-score": 0.8345303406010304, + "support": 42349.0 + }, + "accuracy": 0.9425864835948645, + "macro avg": { + "precision": 0.9314897067914578, + "recall": 0.8751824039060392, + "f1-score": 0.8998989932865968, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9415929605530341, + "recall": 0.9425864835948645, + "f1-score": 0.9405859673649856, + "support": 224320.0 + }, + "cross-entropy": 0.13852674322862857, + "roc-auc": 0.9799226934193556, + "score": 0.9799226934193556 + }, + "val": { + "0": { + "precision": 0.9338218586684844, + "recall": 0.9819604563646138, + "f1-score": 0.9572863590042219, + "support": 16741.0 + }, + "1": { + "precision": 0.8808208366219415, + "recall": 0.6570503385340006, + "f1-score": 0.7526555386949925, + "support": 3397.0 + }, + "accuracy": 0.9271526467375112, + "macro avg": { + "precision": 0.907321347645213, + "recall": 0.8195053974493072, + "f1-score": 0.8549709488496071, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9248813247578624, + "recall": 0.9271526467375112, + "f1-score": 0.9227679909145182, + "support": 20138.0 + }, + "cross-entropy": 0.17399362918195638, + "roc-auc": 0.9588471624971818, + "score": 0.9588471624971818 + }, + "test": { + "0": { + "precision": 0.9311070780399274, + "recall": 0.9755837833726325, + "f1-score": 0.9528266844959513, + "support": 13147.0 + }, + "1": { + "precision": 0.8726190476190476, + "recall": 0.6985387547649301, + "f1-score": 0.7759350741002117, + "support": 3148.0 + }, + "accuracy": 0.9220619822031298, + "macro avg": { + "precision": 0.9018630628294875, + "recall": 0.8370612690687813, + "f1-score": 0.8643808792980815, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9198078868914199, + "recall": 0.9220619822031298, + "f1-score": 0.9186533313492322, + "support": 16295.0 + }, + "cross-entropy": 0.17933181253710723, + "roc-auc": 0.962094141903753, + "score": 0.962094141903753 + } + }, + "time": "0:05:30.729154", + "diversity": { + "train": { + "mean_centered_corr": 0.08804295212030411, + "mean_pairwise_disagreement": 0.03950866022329851, + "member_std": 0.03889348357915878 + }, + "val": { + "mean_centered_corr": 0.0962420254945755, + "mean_pairwise_disagreement": 0.03718395490470592, + "member_std": 0.042661890387535095 + }, + "test": { + "mean_centered_corr": 0.11283686012029648, + "mean_pairwise_disagreement": 0.04520397608607429, + "member_std": 0.04494225233793259 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31, + 4, + 7, + 22, + 16, + 23, + 5, + 30, + 17, + 7 + ], + "head_selection_time": "0:00:11.116484" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..fb3e78cdb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.001-best-head-evaluation/0/report.json @@ -0,0 +1,272 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.001-evaluation/_esam_diagnostics/esam_rho0.001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9505616894590861, + "recall": 0.9755675354864236, + "f1-score": 0.9629022940967485, + "support": 181971.0 + }, + "1": { + "precision": 0.8816356956498589, + "recall": 0.7819783229828331, + "f1-score": 0.8288220645468083, + "support": 42349.0 + }, + "accuracy": 0.93902014978602, + "macro avg": { + "precision": 0.9160986925544725, + "recall": 0.8787729292346284, + "f1-score": 0.8958621793217785, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9375492656367478, + "recall": 0.93902014978602, + "f1-score": 0.9375895103894981, + "support": 224320.0 + }, + "cross-entropy": 0.14190517817089304, + "roc-auc": 0.9769881887024199, + "score": 0.9769881887024199 + }, + "val": { + "0": { + "precision": 0.9410778304614317, + "recall": 0.9721641479003644, + "f1-score": 0.9563684442485677, + "support": 16741.0 + }, + "1": { + "precision": 0.8361462728551337, + "recall": 0.7000294377391816, + "f1-score": 0.7620573626021471, + "support": 3397.0 + }, + "accuracy": 0.9262588141821432, + "macro avg": { + "precision": 0.8886120516582827, + "recall": 0.836096792819773, + "f1-score": 0.8592129034253574, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9233773387945037, + "recall": 0.9262588141821432, + "f1-score": 0.9235908722775235, + "support": 20138.0 + }, + "cross-entropy": 0.17980444186714722, + "roc-auc": 0.9561281764285071, + "score": 0.9561281764285071 + }, + "test": { + "0": { + "precision": 0.937955119631446, + "recall": 0.9601429984026775, + "f1-score": 0.9489193760571322, + "support": 13147.0 + }, + "1": { + "precision": 0.8152978498413818, + "recall": 0.7347522236340533, + "f1-score": 0.7729323308270677, + "support": 3148.0 + }, + "accuracy": 0.9166001841055539, + "macro avg": { + "precision": 0.8766264847364138, + "recall": 0.8474476110183654, + "f1-score": 0.8609258534420999, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.914259195403209, + "recall": 0.9166001841055539, + "f1-score": 0.9149207741311277, + "support": 16295.0 + }, + "cross-entropy": 0.18582539957016014, + "roc-auc": 0.9607344074031798, + "score": 0.9607344074031798 + } + }, + "time": "0:05:03.728563", + "diversity": { + "train": { + "mean_centered_corr": 0.051523078233003616, + "mean_pairwise_disagreement": 0.03397294259928213, + "member_std": 0.03101634979248047 + }, + "val": { + "mean_centered_corr": 0.06029462069272995, + "mean_pairwise_disagreement": 0.032282392619954574, + "member_std": 0.03389967605471611 + }, + "test": { + "mean_centered_corr": 0.07432211190462112, + "mean_pairwise_disagreement": 0.04083245899692168, + "member_std": 0.03589051589369774 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31 + ], + "head_selection_time": "0:00:00.792824" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.001-evaluation/0.toml new file mode 100644 index 000000000..1c9bfed00 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.001-evaluation/0.toml @@ -0,0 +1,44 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.001 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.001-evaluation/0/report.json new file mode 100644 index 000000000..aa3e33961 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.001-evaluation/0/report.json @@ -0,0 +1,268 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.001-evaluation/_esam_diagnostics/esam_rho0.001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.947970671011702, + "recall": 0.9847338312148639, + "f1-score": 0.9660026037806032, + "support": 181971.0 + }, + "1": { + "precision": 0.921285277116627, + "recall": 0.7677631112895228, + "f1-score": 0.8375471722414704, + "support": 42349.0 + }, + "accuracy": 0.9437722895863053, + "macro avg": { + "precision": 0.9346279740641645, + "recall": 0.8762484712521934, + "f1-score": 0.9017748880110368, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9429327798470153, + "recall": 0.9437722895863053, + "f1-score": 0.9417517163418964, + "support": 224320.0 + }, + "cross-entropy": 0.13293877063076864, + "roc-auc": 0.9811870501530091, + "score": 0.9811870501530091 + }, + "val": { + "0": { + "precision": 0.9362066016760732, + "recall": 0.9809449853652709, + "f1-score": 0.9580537891604924, + "support": 16741.0 + }, + "1": { + "precision": 0.8771659607239122, + "recall": 0.6705916985575507, + "f1-score": 0.7600934267600934, + "support": 3397.0 + }, + "accuracy": 0.9285927102989373, + "macro avg": { + "precision": 0.9066862811999927, + "recall": 0.8257683419614108, + "f1-score": 0.859073607960293, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9262472682112558, + "recall": 0.9285927102989373, + "f1-score": 0.9246606343748058, + "support": 20138.0 + }, + "cross-entropy": 0.17347812058673698, + "roc-auc": 0.958366515485181, + "score": 0.958366515485181 + }, + "test": { + "0": { + "precision": 0.9325041821223362, + "recall": 0.9752034684718948, + "f1-score": 0.9533759666864962, + "support": 13147.0 + }, + "1": { + "precision": 0.8719560094265515, + "recall": 0.7052096569250318, + "f1-score": 0.779768177028451, + "support": 3148.0 + }, + "accuracy": 0.9230438784903344, + "macro avg": { + "precision": 0.9022300957744438, + "recall": 0.8402065626984633, + "f1-score": 0.8665720718574736, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9208069960133255, + "recall": 0.9230438784903344, + "f1-score": 0.9198370086107965, + "support": 16295.0 + }, + "cross-entropy": 0.17737713039078654, + "roc-auc": 0.9625169220800972, + "score": 0.9625169220800972 + } + }, + "time": "0:05:03.728563", + "diversity": { + "train": { + "mean_centered_corr": 0.051523078233003616, + "mean_pairwise_disagreement": 0.03397294259928213, + "member_std": 0.03101634979248047 + }, + "val": { + "mean_centered_corr": 0.06029462069272995, + "mean_pairwise_disagreement": 0.032282392619954574, + "member_std": 0.03389967605471611 + }, + "test": { + "mean_centered_corr": 0.07432211190462112, + "mean_pairwise_disagreement": 0.04083245899692168, + "member_std": 0.03589051589369774 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..812f0329f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,285 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.001-evaluation/_esam_diagnostics/esam_rho0.001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9463750527648797, + "recall": 0.9856350737205379, + "f1-score": 0.9656061654252728, + "support": 181971.0 + }, + "1": { + "precision": 0.9248850574712644, + "recall": 0.7600179461144301, + "f1-score": 0.8343854100506811, + "support": 42349.0 + }, + "accuracy": 0.9430411911554921, + "macro avg": { + "precision": 0.9356300551180721, + "recall": 0.872826509917484, + "f1-score": 0.899995787737977, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9423179922678696, + "recall": 0.9430411911554921, + "f1-score": 0.9408332170953934, + "support": 224320.0 + }, + "cross-entropy": 0.13463392233996885, + "roc-auc": 0.9802844103471857, + "score": 0.9802844103471857 + }, + "val": { + "0": { + "precision": 0.9344280925052559, + "recall": 0.9823188578937937, + "f1-score": 0.9577751892836343, + "support": 16741.0 + }, + "1": { + "precision": 0.8834186687672312, + "recall": 0.66028848984398, + "f1-score": 0.7557277628032345, + "support": 3397.0 + }, + "accuracy": 0.927996821928692, + "macro avg": { + "precision": 0.9089233806362436, + "recall": 0.8213036738688868, + "f1-score": 0.8567514760434344, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9258235134786361, + "recall": 0.927996821928692, + "f1-score": 0.9236926037362155, + "support": 20138.0 + }, + "cross-entropy": 0.17320355151838052, + "roc-auc": 0.9589766350935588, + "score": 0.9589766350935588 + }, + "test": { + "0": { + "precision": 0.9308371958285052, + "recall": 0.9776374838366167, + "f1-score": 0.9536635132628455, + "support": 13147.0 + }, + "1": { + "precision": 0.8817852834740652, + "recall": 0.6966327827191868, + "f1-score": 0.7783496007098492, + "support": 3148.0 + }, + "accuracy": 0.9233507210800859, + "macro avg": { + "precision": 0.9063112396512851, + "recall": 0.8371351332779018, + "f1-score": 0.8660065569863473, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9213609503488013, + "recall": 0.9233507210800859, + "f1-score": 0.9197949525560746, + "support": 16295.0 + }, + "cross-entropy": 0.17676394803557405, + "roc-auc": 0.962915757398333, + "score": 0.962915757398333 + } + }, + "time": "0:05:03.728563", + "diversity": { + "train": { + "mean_centered_corr": 0.051523078233003616, + "mean_pairwise_disagreement": 0.03397294259928213, + "member_std": 0.03101634979248047 + }, + "val": { + "mean_centered_corr": 0.06029462069272995, + "mean_pairwise_disagreement": 0.032282392619954574, + "member_std": 0.03389967605471611 + }, + "test": { + "mean_centered_corr": 0.07432211190462112, + "mean_pairwise_disagreement": 0.04083245899692168, + "member_std": 0.03589051589369774 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31, + 4, + 26, + 2, + 24, + 9, + 12, + 7, + 28, + 15, + 26, + 17, + 2, + 3 + ], + "head_selection_time": "0:00:13.310784" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.0025-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.0025-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.0025-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.0025-best-head-evaluation/0/report.json new file mode 100644 index 000000000..3d42603b9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.0025-best-head-evaluation/0/report.json @@ -0,0 +1,272 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.0025-evaluation/_esam_diagnostics/esam_rho0.0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9320793769889524, + "recall": 0.9898775079545642, + "f1-score": 0.9601093740838854, + "support": 181971.0 + }, + "1": { + "precision": 0.9407049734427813, + "recall": 0.6900517131455288, + "f1-score": 0.7961151823902798, + "support": 42349.0 + }, + "accuracy": 0.9332738944365193, + "macro avg": { + "precision": 0.9363921752158668, + "recall": 0.8399646105500465, + "f1-score": 0.8781122782370826, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9337077890084923, + "recall": 0.9332738944365193, + "f1-score": 0.9291491831778916, + "support": 224320.0 + }, + "cross-entropy": 0.14914181629256673, + "roc-auc": 0.9749246612138766, + "score": 0.9749246612138766 + }, + "val": { + "0": { + "precision": 0.9254064927082751, + "recall": 0.9893076877128009, + "f1-score": 0.9562907789133321, + "support": 16741.0 + }, + "1": { + "precision": 0.9201249442213297, + "recall": 0.6070061819252281, + "f1-score": 0.7314650585313941, + "support": 3397.0 + }, + "accuracy": 0.9248187506207171, + "macro avg": { + "precision": 0.9227657184648024, + "recall": 0.7981569348190145, + "f1-score": 0.8438779187223631, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9245155690708656, + "recall": 0.9248187506207171, + "f1-score": 0.9183658125741999, + "support": 20138.0 + }, + "cross-entropy": 0.18253383914635615, + "roc-auc": 0.9561005955827354, + "score": 0.9561005955827354 + }, + "test": { + "0": { + "precision": 0.9195605953224664, + "recall": 0.9869171674146193, + "f1-score": 0.952049014931944, + "support": 13147.0 + }, + "1": { + "precision": 0.9212814645308924, + "recall": 0.6394536213468869, + "f1-score": 0.7549221826364148, + "support": 3148.0 + }, + "accuracy": 0.919791347038969, + "macro avg": { + "precision": 0.9204210299266794, + "recall": 0.8131853943807531, + "f1-score": 0.8534855987841794, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9198930467657388, + "recall": 0.919791347038969, + "f1-score": 0.9139664578244677, + "support": 16295.0 + }, + "cross-entropy": 0.1878335545651875, + "roc-auc": 0.9607496659076156, + "score": 0.9607496659076156 + } + }, + "time": "0:04:50.502999", + "diversity": { + "train": { + "mean_centered_corr": 0.055189475417137146, + "mean_pairwise_disagreement": 0.03156465166409737, + "member_std": 0.02923482470214367 + }, + "val": { + "mean_centered_corr": 0.06409875303506851, + "mean_pairwise_disagreement": 0.02878024694126655, + "member_std": 0.03127071633934975 + }, + "test": { + "mean_centered_corr": 0.07738936692476273, + "mean_pairwise_disagreement": 0.03777504478911996, + "member_std": 0.03404448181390762 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.834549" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.0025-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.0025-evaluation/0.toml new file mode 100644 index 000000000..a0ce28efa --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.0025-evaluation/0.toml @@ -0,0 +1,44 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.0025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.0025-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.0025-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.0025-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.0025-evaluation/0/report.json new file mode 100644 index 000000000..779546f37 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.0025-evaluation/0/report.json @@ -0,0 +1,268 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.0025-evaluation/_esam_diagnostics/esam_rho0.0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9447991304623112, + "recall": 0.9840249270488155, + "f1-score": 0.9640131683432168, + "support": 181971.0 + }, + "1": { + "precision": 0.9164511122607346, + "recall": 0.7529575668846962, + "f1-score": 0.8266984690769091, + "support": 42349.0 + }, + "accuracy": 0.9404021041369472, + "macro avg": { + "precision": 0.9306251213615229, + "recall": 0.8684912469667558, + "f1-score": 0.895355818710063, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9394473552179345, + "recall": 0.9404021041369472, + "f1-score": 0.9380897544736159, + "support": 224320.0 + }, + "cross-entropy": 0.14008234966516878, + "roc-auc": 0.978457814057529, + "score": 0.978457814057529 + }, + "val": { + "0": { + "precision": 0.9334657961964234, + "recall": 0.9821993907174004, + "f1-score": 0.9572127139364304, + "support": 16741.0 + }, + "1": { + "precision": 0.8818866428854538, + "recall": 0.6549896967912864, + "f1-score": 0.7516891891891891, + "support": 3397.0 + }, + "accuracy": 0.9270036746449498, + "macro avg": { + "precision": 0.9076762195409387, + "recall": 0.8185945437543434, + "f1-score": 0.8544509515628098, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9247651117293779, + "recall": 0.9270036746449498, + "f1-score": 0.9225437590468496, + "support": 20138.0 + }, + "cross-entropy": 0.17400317835615806, + "roc-auc": 0.9581563313286563, + "score": 0.9581563313286563 + }, + "test": { + "0": { + "precision": 0.9298753984352361, + "recall": 0.9763444131741081, + "f1-score": 0.9525435048792252, + "support": 13147.0 + }, + "1": { + "precision": 0.8751505419510237, + "recall": 0.6925031766200762, + "f1-score": 0.7731867352367441, + "support": 3148.0 + }, + "accuracy": 0.9215096655415772, + "macro avg": { + "precision": 0.9025129701931299, + "recall": 0.8344237948970922, + "f1-score": 0.8628651200579847, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9193032076888538, + "recall": 0.9215096655415772, + "f1-score": 0.9178939123149706, + "support": 16295.0 + }, + "cross-entropy": 0.17809609825505548, + "roc-auc": 0.96205585670933, + "score": 0.96205585670933 + } + }, + "time": "0:04:50.502999", + "diversity": { + "train": { + "mean_centered_corr": 0.055189475417137146, + "mean_pairwise_disagreement": 0.03156465166409737, + "member_std": 0.02923482470214367 + }, + "val": { + "mean_centered_corr": 0.06409875303506851, + "mean_pairwise_disagreement": 0.02878024694126655, + "member_std": 0.03127071633934975 + }, + "test": { + "mean_centered_corr": 0.07738936692476273, + "mean_pairwise_disagreement": 0.03777504478911996, + "member_std": 0.03404448181390762 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.0025-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.0025-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.0025-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.0025-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..59be8f2e4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.0025-greedy-heads-evaluation/0/report.json @@ -0,0 +1,281 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.0025-evaluation/_esam_diagnostics/esam_rho0.0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9408421041602272, + "recall": 0.9868990113809343, + "f1-score": 0.963320370122033, + "support": 181971.0 + }, + "1": { + "precision": 0.9287102658413325, + "recall": 0.733358520862358, + "f1-score": 0.8195540308747856, + "support": 42349.0 + }, + "accuracy": 0.9390335235378031, + "macro avg": { + "precision": 0.9347761850007799, + "recall": 0.8601287661216461, + "f1-score": 0.8914372004984092, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9385517545660453, + "recall": 0.9390335235378031, + "f1-score": 0.9361789618624856, + "support": 224320.0 + }, + "cross-entropy": 0.1413648202791101, + "roc-auc": 0.9778507602646593, + "score": 0.9778507602646593 + }, + "val": { + "0": { + "precision": 0.9304607749139925, + "recall": 0.9854847380682158, + "f1-score": 0.957182640983987, + "support": 16741.0 + }, + "1": { + "precision": 0.8990444536767761, + "recall": 0.6370326758904916, + "f1-score": 0.7456926257753274, + "support": 3397.0 + }, + "accuracy": 0.9267057304598272, + "macro avg": { + "precision": 0.9147526142953843, + "recall": 0.8112587069793538, + "f1-score": 0.8514376333796572, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9251612792717825, + "recall": 0.9267057304598272, + "f1-score": 0.9215072222897861, + "support": 20138.0 + }, + "cross-entropy": 0.17378940292970774, + "roc-auc": 0.95870368407125, + "score": 0.95870368407125 + }, + "test": { + "0": { + "precision": 0.9256684108666046, + "recall": 0.982277325625618, + "f1-score": 0.9531330725514798, + "support": 13147.0 + }, + "1": { + "precision": 0.9005972696245734, + "recall": 0.670584498094028, + "f1-score": 0.7687545520757465, + "support": 3148.0 + }, + "accuracy": 0.9220619822031298, + "macro avg": { + "precision": 0.9131328402455889, + "recall": 0.826430911859823, + "f1-score": 0.8609438123136132, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9208249648629276, + "recall": 0.9220619822031298, + "f1-score": 0.9175133375126576, + "support": 16295.0 + }, + "cross-entropy": 0.17801794809138968, + "roc-auc": 0.962465432178352, + "score": 0.962465432178352 + } + }, + "time": "0:04:50.502999", + "diversity": { + "train": { + "mean_centered_corr": 0.055189475417137146, + "mean_pairwise_disagreement": 0.03156465166409737, + "member_std": 0.02923482470214367 + }, + "val": { + "mean_centered_corr": 0.06409875303506851, + "mean_pairwise_disagreement": 0.02878024694126655, + "member_std": 0.03127071633934975 + }, + "test": { + "mean_centered_corr": 0.07738936692476273, + "mean_pairwise_disagreement": 0.03777504478911996, + "member_std": 0.03404448181390762 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 13, + 28, + 5, + 30, + 12, + 21, + 4, + 17, + 29, + 31 + ], + "head_selection_time": "0:00:10.965687" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..5e67db395 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.005-best-head-evaluation/0/report.json @@ -0,0 +1,272 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.005-evaluation/_esam_diagnostics/esam_rho0.005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9545321160743397, + "recall": 0.9652801820070231, + "f1-score": 0.9598760625261277, + "support": 181971.0 + }, + "1": { + "precision": 0.8432258064516129, + "recall": 0.8024274481097546, + "f1-score": 0.8223208992244311, + "support": 42349.0 + }, + "accuracy": 0.934535485021398, + "macro avg": { + "precision": 0.8988789612629764, + "recall": 0.8838538150583889, + "f1-score": 0.8910984808752793, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9335187828619073, + "recall": 0.934535485021398, + "f1-score": 0.9339072518509157, + "support": 224320.0 + }, + "cross-entropy": 0.1498442336366005, + "roc-auc": 0.9747257011404723, + "score": 0.9747257011404723 + }, + "val": { + "0": { + "precision": 0.9430700273589848, + "recall": 0.9677438623738128, + "f1-score": 0.955247641509434, + "support": 16741.0 + }, + "1": { + "precision": 0.8175059141601893, + "recall": 0.7120989108036503, + "f1-score": 0.7611705475141598, + "support": 3397.0 + }, + "accuracy": 0.9246201211639686, + "macro avg": { + "precision": 0.880287970759587, + "recall": 0.8399213865887316, + "f1-score": 0.8582090945117968, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9218891110546691, + "recall": 0.9246201211639686, + "f1-score": 0.9225095399947877, + "support": 20138.0 + }, + "cross-entropy": 0.18072654373317865, + "roc-auc": 0.9561512908829329, + "score": 0.9561512908829329 + }, + "test": { + "0": { + "precision": 0.938474098263322, + "recall": 0.9618163839659238, + "f1-score": 0.9500018782164457, + "support": 13147.0 + }, + "1": { + "precision": 0.8220489188231124, + "recall": 0.7366581956797967, + "f1-score": 0.7770145753057464, + "support": 3148.0 + }, + "accuracy": 0.9183185026081621, + "macro avg": { + "precision": 0.8802615085432172, + "recall": 0.8492372898228603, + "f1-score": 0.8635082267610961, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9159821396945721, + "recall": 0.9183185026081621, + "f1-score": 0.9165827907931329, + "support": 16295.0 + }, + "cross-entropy": 0.18568286621420937, + "roc-auc": 0.9606910602029306, + "score": 0.9606910602029306 + } + }, + "time": "0:05:03.738226", + "diversity": { + "train": { + "mean_centered_corr": 0.049995824694633484, + "mean_pairwise_disagreement": 0.03397909919872533, + "member_std": 0.031108465045690536 + }, + "val": { + "mean_centered_corr": 0.05969587340950966, + "mean_pairwise_disagreement": 0.03266793800197989, + "member_std": 0.03460051864385605 + }, + "test": { + "mean_centered_corr": 0.07376471161842346, + "mean_pairwise_disagreement": 0.04182474833958566, + "member_std": 0.03703681379556656 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 17 + ], + "head_selection_time": "0:00:00.955041" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.005-evaluation/0.toml new file mode 100644 index 000000000..016b99992 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.005-evaluation/0.toml @@ -0,0 +1,44 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.005-evaluation/0/report.json new file mode 100644 index 000000000..382e4333a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.005-evaluation/0/report.json @@ -0,0 +1,268 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.005-evaluation/_esam_diagnostics/esam_rho0.005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9496102021429594, + "recall": 0.9833215182638992, + "f1-score": 0.9661718889206862, + "support": 181971.0 + }, + "1": { + "precision": 0.9154336983476832, + "recall": 0.7757916361661432, + "f1-score": 0.8398476443671873, + "support": 42349.0 + }, + "accuracy": 0.9441422967189729, + "macro avg": { + "precision": 0.9325219502453213, + "recall": 0.8795565772150212, + "f1-score": 0.9030097666439367, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9431580767897758, + "recall": 0.9441422967189729, + "f1-score": 0.9423233447311528, + "support": 224320.0 + }, + "cross-entropy": 0.13329761494760606, + "roc-auc": 0.9810524262916843, + "score": 0.9810524262916843 + }, + "val": { + "0": { + "precision": 0.936472602739726, + "recall": 0.9800489815423212, + "f1-score": 0.9577653892180613, + "support": 16741.0 + }, + "1": { + "precision": 0.8724216959511077, + "recall": 0.6723579629084486, + "f1-score": 0.7594347464671655, + "support": 3397.0 + }, + "accuracy": 0.9281457940212533, + "macro avg": { + "precision": 0.9044471493454169, + "recall": 0.8262034722253849, + "f1-score": 0.8586000678426133, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9256681072406231, + "recall": 0.9281457940212533, + "f1-score": 0.9243097732966793, + "support": 20138.0 + }, + "cross-entropy": 0.1734408298186842, + "roc-auc": 0.9583120395781356, + "score": 0.9583120395781356 + }, + "test": { + "0": { + "precision": 0.9324796274738067, + "recall": 0.9748231535711569, + "f1-score": 0.9531813617939088, + "support": 13147.0 + }, + "1": { + "precision": 0.8702469619756958, + "recall": 0.7052096569250318, + "f1-score": 0.779084049833304, + "support": 3148.0 + }, + "accuracy": 0.922737035900583, + "macro avg": { + "precision": 0.9013632947247513, + "recall": 0.8400164052480943, + "f1-score": 0.8661327058136064, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9204570174101029, + "recall": 0.922737035900583, + "f1-score": 0.9195478338373586, + "support": 16295.0 + }, + "cross-entropy": 0.17694458755574416, + "roc-auc": 0.9627483366901237, + "score": 0.9627483366901237 + } + }, + "time": "0:05:03.738226", + "diversity": { + "train": { + "mean_centered_corr": 0.049995824694633484, + "mean_pairwise_disagreement": 0.03397909919872533, + "member_std": 0.031108465045690536 + }, + "val": { + "mean_centered_corr": 0.05969587340950966, + "mean_pairwise_disagreement": 0.03266793800197989, + "member_std": 0.03460051864385605 + }, + "test": { + "mean_centered_corr": 0.07376471161842346, + "mean_pairwise_disagreement": 0.04182474833958566, + "member_std": 0.03703681379556656 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..06391c184 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,290 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.005-evaluation/_esam_diagnostics/esam_rho0.005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9495084683651196, + "recall": 0.9824752295695468, + "f1-score": 0.9657105817533626, + "support": 181971.0 + }, + "1": { + "precision": 0.9114928811301379, + "recall": 0.7755082764646155, + "f1-score": 0.8380199030364889, + "support": 42349.0 + }, + "accuracy": 0.9434022824536377, + "macro avg": { + "precision": 0.9305006747476288, + "recall": 0.8789917530170812, + "f1-score": 0.9018652423949257, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9423315688295711, + "recall": 0.9434022824536377, + "f1-score": 0.9416040707290184, + "support": 224320.0 + }, + "cross-entropy": 0.13410595787789584, + "roc-auc": 0.9804656245919036, + "score": 0.9804656245919036 + }, + "val": { + "0": { + "precision": 0.9362795477903392, + "recall": 0.9795113792485515, + "f1-score": 0.957407677711283, + "support": 16741.0 + }, + "1": { + "precision": 0.8692835365853658, + "recall": 0.6714748307329997, + "f1-score": 0.7576814482644079, + "support": 3397.0 + }, + "accuracy": 0.927549905651008, + "macro avg": { + "precision": 0.9027815421878524, + "recall": 0.8254931049907757, + "f1-score": 0.8575445629878454, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9249782542128591, + "recall": 0.927549905651008, + "f1-score": 0.9237166457601939, + "support": 20138.0 + }, + "cross-entropy": 0.17278545942960094, + "roc-auc": 0.958991133281215, + "score": 0.958991133281215 + }, + "test": { + "0": { + "precision": 0.9319371727748691, + "recall": 0.9748231535711569, + "f1-score": 0.9528978772445073, + "support": 13147.0 + }, + "1": { + "precision": 0.8698387731026347, + "recall": 0.7026683608640406, + "f1-score": 0.7773677736777368, + "support": 3148.0 + }, + "accuracy": 0.9222460877569807, + "macro avg": { + "precision": 0.9008879729387519, + "recall": 0.8387457572175987, + "f1-score": 0.865132825461122, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9199405012702239, + "recall": 0.9222460877569807, + "f1-score": 0.9189875510077357, + "support": 16295.0 + }, + "cross-entropy": 0.1769733127255982, + "roc-auc": 0.9627938464179217, + "score": 0.9627938464179217 + } + }, + "time": "0:05:03.738226", + "diversity": { + "train": { + "mean_centered_corr": 0.049995824694633484, + "mean_pairwise_disagreement": 0.03397909919872533, + "member_std": 0.031108465045690536 + }, + "val": { + "mean_centered_corr": 0.05969587340950966, + "mean_pairwise_disagreement": 0.03266793800197989, + "member_std": 0.03460051864385605 + }, + "test": { + "mean_centered_corr": 0.07376471161842346, + "mean_pairwise_disagreement": 0.04182474833958566, + "member_std": 0.03703681379556656 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 17, + 23, + 5, + 26, + 21, + 24, + 30, + 9, + 28, + 15, + 22, + 12, + 20, + 29, + 23, + 26, + 5, + 21, + 30 + ], + "head_selection_time": "0:00:17.296804" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..69dc71c4f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.01-best-head-evaluation/0/report.json @@ -0,0 +1,272 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.01-evaluation/_esam_diagnostics/esam_rho0.01-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9467367680900025, + "recall": 0.976688593237384, + "f1-score": 0.9614794738422339, + "support": 181971.0 + }, + "1": { + "precision": 0.8840730214254482, + "recall": 0.7638905287019764, + "f1-score": 0.8195994476887802, + "support": 42349.0 + }, + "accuracy": 0.9365148002853068, + "macro avg": { + "precision": 0.9154048947577254, + "recall": 0.8702895609696801, + "f1-score": 0.890539460765507, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9349065834988058, + "recall": 0.9365148002853068, + "f1-score": 0.9346941794967782, + "support": 224320.0 + }, + "cross-entropy": 0.14560289443931593, + "roc-auc": 0.9752118908061648, + "score": 0.9752118908061648 + }, + "val": { + "0": { + "precision": 0.9361300291878899, + "recall": 0.9770623021324891, + "f1-score": 0.9561582977728416, + "support": 16741.0 + }, + "1": { + "precision": 0.8559099437148218, + "recall": 0.6714748307329997, + "f1-score": 0.7525569119102606, + "support": 3397.0 + }, + "accuracy": 0.9255139537193365, + "macro avg": { + "precision": 0.8960199864513558, + "recall": 0.8242685664327445, + "f1-score": 0.8543576048415511, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9225980185437339, + "recall": 0.9255139537193365, + "f1-score": 0.9218135809302958, + "support": 20138.0 + }, + "cross-entropy": 0.18142101466285146, + "roc-auc": 0.9554589123032324, + "score": 0.9554589123032324 + }, + "test": { + "0": { + "precision": 0.9283744557329463, + "recall": 0.973073705027763, + "f1-score": 0.9501986853344226, + "support": 13147.0 + }, + "1": { + "precision": 0.8592445328031809, + "recall": 0.6864675984752223, + "f1-score": 0.7631997174642415, + "support": 3148.0 + }, + "accuracy": 0.9177048174286591, + "macro avg": { + "precision": 0.8938094942680637, + "recall": 0.8297706517514927, + "f1-score": 0.8566992013993321, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9150193776486933, + "recall": 0.9177048174286591, + "f1-score": 0.91407271105671, + "support": 16295.0 + }, + "cross-entropy": 0.18737908577750867, + "roc-auc": 0.9585579261153012, + "score": 0.9585579261153012 + } + }, + "time": "0:05:01.917656", + "diversity": { + "train": { + "mean_centered_corr": 0.049148593097925186, + "mean_pairwise_disagreement": 0.03323895011734389, + "member_std": 0.030483398586511612 + }, + "val": { + "mean_centered_corr": 0.056430645287036896, + "mean_pairwise_disagreement": 0.03106348453733753, + "member_std": 0.03352737799286842 + }, + "test": { + "mean_centered_corr": 0.07052021473646164, + "mean_pairwise_disagreement": 0.039438181116313135, + "member_std": 0.03553345799446106 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 26 + ], + "head_selection_time": "0:00:00.710724" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.01-evaluation/0.toml new file mode 100644 index 000000000..a6e4b28be --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.01-evaluation/0.toml @@ -0,0 +1,44 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.01 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.01-evaluation/0/report.json new file mode 100644 index 000000000..73f544f35 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.01-evaluation/0/report.json @@ -0,0 +1,268 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.01-evaluation/_esam_diagnostics/esam_rho0.01-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9491977758441393, + "recall": 0.9831291799242736, + "f1-score": 0.9658655624151363, + "support": 181971.0 + }, + "1": { + "precision": 0.9143510768887401, + "recall": 0.7739025714892913, + "f1-score": 0.8382847569475529, + "support": 42349.0 + }, + "accuracy": 0.9436296362339515, + "macro avg": { + "precision": 0.9317744263664397, + "recall": 0.8785158757067825, + "f1-score": 0.9020751596813446, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9426191254604812, + "recall": 0.9436296362339515, + "f1-score": 0.9417797941789261, + "support": 224320.0 + }, + "cross-entropy": 0.13432585053967652, + "roc-auc": 0.9806862893380656, + "score": 0.9806862893380656 + }, + "val": { + "0": { + "precision": 0.936262482168331, + "recall": 0.9801087151305179, + "f1-score": 0.957684001634273, + "support": 16741.0 + }, + "1": { + "precision": 0.8725602755453502, + "recall": 0.6711804533411834, + "f1-score": 0.7587354409317804, + "support": 3397.0 + }, + "accuracy": 0.927996821928692, + "macro avg": { + "precision": 0.9044113788568406, + "recall": 0.8256445842358506, + "f1-score": 0.8582097212830266, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9255168075284329, + "recall": 0.927996821928692, + "f1-score": 0.9241241515644365, + "support": 20138.0 + }, + "cross-entropy": 0.17398384502840505, + "roc-auc": 0.9580370399944422, + "score": 0.9580370399944422 + }, + "test": { + "0": { + "precision": 0.9325081907535493, + "recall": 0.9742146497299764, + "f1-score": 0.9529052897849862, + "support": 13147.0 + }, + "1": { + "precision": 0.867578125, + "recall": 0.7055273189326556, + "f1-score": 0.7782060266292922, + "support": 3148.0 + }, + "accuracy": 0.9223074562749309, + "macro avg": { + "precision": 0.9000431578767747, + "recall": 0.839870984331316, + "f1-score": 0.8655556582071392, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9199644750743733, + "recall": 0.9223074562749309, + "f1-score": 0.9191554720240703, + "support": 16295.0 + }, + "cross-entropy": 0.17817953563365546, + "roc-auc": 0.9621106810110944, + "score": 0.9621106810110944 + } + }, + "time": "0:05:01.917656", + "diversity": { + "train": { + "mean_centered_corr": 0.049148593097925186, + "mean_pairwise_disagreement": 0.03323895011734389, + "member_std": 0.030483398586511612 + }, + "val": { + "mean_centered_corr": 0.056430645287036896, + "mean_pairwise_disagreement": 0.03106348453733753, + "member_std": 0.03352737799286842 + }, + "test": { + "mean_centered_corr": 0.07052021473646164, + "mean_pairwise_disagreement": 0.039438181116313135, + "member_std": 0.03553345799446106 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..999ba0393 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,282 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.01-evaluation/_esam_diagnostics/esam_rho0.01-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9505159911985807, + "recall": 0.9804254524072517, + "f1-score": 0.9652390793901555, + "support": 181971.0 + }, + "1": { + "precision": 0.9027387161073642, + "recall": 0.7806795910174975, + "f1-score": 0.8372841006939168, + "support": 42349.0 + }, + "accuracy": 0.942715763195435, + "macro avg": { + "precision": 0.9266273536529724, + "recall": 0.8805525217123746, + "f1-score": 0.9012615900420362, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9414961988357154, + "recall": 0.942715763195435, + "f1-score": 0.9410826716119502, + "support": 224320.0 + }, + "cross-entropy": 0.1373233192307072, + "roc-auc": 0.9795565058577262, + "score": 0.9795565058577262 + }, + "val": { + "0": { + "precision": 0.9383593033913841, + "recall": 0.9784361746610118, + "f1-score": 0.9579787700675498, + "support": 16741.0 + }, + "1": { + "precision": 0.8653989560029829, + "recall": 0.6832499264056521, + "f1-score": 0.7636124362559632, + "support": 3397.0 + }, + "accuracy": 0.9286423676631245, + "macro avg": { + "precision": 0.9018791296971835, + "recall": 0.830843050533332, + "f1-score": 0.8607956031617565, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9260519094059635, + "recall": 0.9286423676631245, + "f1-score": 0.9251918778261178, + "support": 20138.0 + }, + "cross-entropy": 0.17333139914835735, + "roc-auc": 0.9587733087820138, + "score": 0.9587733087820138 + }, + "test": { + "0": { + "precision": 0.9348479296445584, + "recall": 0.9702593747623032, + "f1-score": 0.9522245446401911, + "support": 13147.0 + }, + "1": { + "precision": 0.8524528301886792, + "recall": 0.7175984752223634, + "f1-score": 0.7792342186961021, + "support": 3148.0 + }, + "accuracy": 0.9214482970236268, + "macro avg": { + "precision": 0.8936503799166189, + "recall": 0.8439289249923333, + "f1-score": 0.8657293816681466, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9189301773839197, + "recall": 0.9214482970236268, + "f1-score": 0.9188048732028181, + "support": 16295.0 + }, + "cross-entropy": 0.17776343990066415, + "roc-auc": 0.9626104060922291, + "score": 0.9626104060922291 + } + }, + "time": "0:05:01.917656", + "diversity": { + "train": { + "mean_centered_corr": 0.049148593097925186, + "mean_pairwise_disagreement": 0.03323895011734389, + "member_std": 0.030483398586511612 + }, + "val": { + "mean_centered_corr": 0.056430645287036896, + "mean_pairwise_disagreement": 0.03106348453733753, + "member_std": 0.03352737799286842 + }, + "test": { + "mean_centered_corr": 0.07052021473646164, + "mean_pairwise_disagreement": 0.039438181116313135, + "member_std": 0.03553345799446106 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 26, + 30, + 24, + 5, + 17, + 31, + 22, + 13, + 15, + 20, + 24 + ], + "head_selection_time": "0:00:11.234254" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_cf_fisd-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_cf_fisd-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_cf_fisd-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_cf_fisd-best-head-evaluation/0/report.json new file mode 100644 index 000000000..4003e9150 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_cf_fisd-best-head-evaluation/0/report.json @@ -0,0 +1,657 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.901321291923523, + "min_features_per_member": 256.0, + "mean_features_per_member": 268.59375, + "max_features_per_member": 279.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.842281341552734, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8197222809700668, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9432566752157673, + "recall": 0.9741662132977232, + "f1-score": 0.9584623078898636, + "support": 181971.0 + }, + "1": { + "precision": 0.8708019567965701, + "recall": 0.7481876785756453, + "f1-score": 0.8048517177875151, + "support": 42349.0 + }, + "accuracy": 0.9315041012838802, + "macro avg": { + "precision": 0.9070293160061687, + "recall": 0.8611769459366843, + "f1-score": 0.8816570128386894, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9295780693387408, + "recall": 0.9315041012838802, + "f1-score": 0.9294624198716558, + "support": 224320.0 + }, + "cross-entropy": 0.15818911070182506, + "roc-auc": 0.9710347146934778, + "score": 0.9710347146934778 + }, + "val": { + "0": { + "precision": 0.9363334478415206, + "recall": 0.9768831013678991, + "f1-score": 0.9561785599438711, + "support": 16741.0 + }, + "1": { + "precision": 0.8551646706586826, + "recall": 0.672652340300265, + "f1-score": 0.753007085187016, + "support": 3397.0 + }, + "accuracy": 0.9255636110835237, + "macro avg": { + "precision": 0.8957490592501016, + "recall": 0.824767720834082, + "f1-score": 0.8545928225654436, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9226414061248607, + "recall": 0.9255636110835237, + "f1-score": 0.9219063631145417, + "support": 20138.0 + }, + "cross-entropy": 0.1787492077720551, + "roc-auc": 0.9576967765860231, + "score": 0.9576967765860231 + }, + "test": { + "0": { + "precision": 0.9309967939376275, + "recall": 0.971856697345402, + "f1-score": 0.9509880540359495, + "support": 13147.0 + }, + "1": { + "precision": 0.8560871256320498, + "recall": 0.6991740787801779, + "f1-score": 0.7697149851372618, + "support": 3148.0 + }, + "accuracy": 0.9191776618594661, + "macro avg": { + "precision": 0.8935419597848386, + "recall": 0.8355153880627899, + "f1-score": 0.8603515195866056, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9165251378574827, + "recall": 0.9191776618594661, + "f1-score": 0.915968255269882, + "support": 16295.0 + }, + "cross-entropy": 0.18291214189215663, + "roc-auc": 0.9617315500639867, + "score": 0.9617315500639867 + } + }, + "time": "0:03:52.600341", + "diversity": { + "train": { + "mean_centered_corr": 0.0775546282529831, + "mean_pairwise_disagreement": 0.0356185432101606, + "member_std": 0.03550577908754349 + }, + "val": { + "mean_centered_corr": 0.08212897181510925, + "mean_pairwise_disagreement": 0.03171593825186856, + "member_std": 0.03616226091980934 + }, + "test": { + "mean_centered_corr": 0.10865671932697296, + "mean_pairwise_disagreement": 0.038570855892862445, + "member_std": 0.038954634219408035 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31 + ], + "head_selection_time": "0:00:01.289307" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_cf_fisd-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_cf_fisd-evaluation/0.toml new file mode 100644 index 000000000..10f998b1d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_cf_fisd-evaluation/0.toml @@ -0,0 +1,58 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 15 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "homesite-insurance" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_cf_fisd-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_cf_fisd-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_cf_fisd-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_cf_fisd-evaluation/0/report.json new file mode 100644 index 000000000..4fb8df0ec --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_cf_fisd-evaluation/0/report.json @@ -0,0 +1,653 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.901321291923523, + "min_features_per_member": 256.0, + "mean_features_per_member": 268.59375, + "max_features_per_member": 279.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.842281341552734, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8197222809700668, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9398710962062461, + "recall": 0.9856790367695951, + "f1-score": 0.9622301917057169, + "support": 181971.0 + }, + "1": { + "precision": 0.9221624850657109, + "recall": 0.7290372854140593, + "f1-score": 0.8143058724234792, + "support": 42349.0 + }, + "accuracy": 0.9372280670470756, + "macro avg": { + "precision": 0.9310167906359785, + "recall": 0.8573581610918273, + "f1-score": 0.888268032064598, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.93652791693917, + "recall": 0.9372280670470756, + "f1-score": 0.9343038053055588, + "support": 224320.0 + }, + "cross-entropy": 0.14711870269948546, + "roc-auc": 0.9762437618004894, + "score": 0.9762437618004894 + }, + "val": { + "0": { + "precision": 0.9303347987825499, + "recall": 0.9859626067737889, + "f1-score": 0.9573412986109098, + "support": 16741.0 + }, + "1": { + "precision": 0.9019198664440734, + "recall": 0.6361495437150427, + "f1-score": 0.7460728465389262, + "support": 3397.0 + }, + "accuracy": 0.9269540172807628, + "macro avg": { + "precision": 0.9161273326133117, + "recall": 0.8110560752444158, + "f1-score": 0.8517070725749181, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9255415956266354, + "recall": 0.9269540172807628, + "f1-score": 0.9217032545306373, + "support": 20138.0 + }, + "cross-entropy": 0.175030748761162, + "roc-auc": 0.9589238560635404, + "score": 0.9589238560635404 + }, + "test": { + "0": { + "precision": 0.925214899713467, + "recall": 0.9824294515859131, + "f1-score": 0.9529641789943557, + "support": 13147.0 + }, + "1": { + "precision": 0.9010706638115632, + "recall": 0.6683608640406608, + "f1-score": 0.7674630676636878, + "support": 3148.0 + }, + "accuracy": 0.9217551396133783, + "macro avg": { + "precision": 0.9131427817625151, + "recall": 0.825395157813287, + "f1-score": 0.8602136233290217, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9205505207862382, + "recall": 0.9217551396133783, + "f1-score": 0.9171275727673571, + "support": 16295.0 + }, + "cross-entropy": 0.17966720795996355, + "roc-auc": 0.9624954901031625, + "score": 0.9624954901031625 + } + }, + "time": "0:03:52.600341", + "diversity": { + "train": { + "mean_centered_corr": 0.0775546282529831, + "mean_pairwise_disagreement": 0.0356185432101606, + "member_std": 0.03550577908754349 + }, + "val": { + "mean_centered_corr": 0.08212897181510925, + "mean_pairwise_disagreement": 0.03171593825186856, + "member_std": 0.03616226091980934 + }, + "test": { + "mean_centered_corr": 0.10865671932697296, + "mean_pairwise_disagreement": 0.038570855892862445, + "member_std": 0.038954634219408035 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_cf_fisd-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_cf_fisd-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_cf_fisd-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_cf_fisd-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..fb5104edb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_cf_fisd-greedy-heads-evaluation/0/report.json @@ -0,0 +1,666 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.901321291923523, + "min_features_per_member": 256.0, + "mean_features_per_member": 268.59375, + "max_features_per_member": 279.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.842281341552734, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8197222809700668, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9379184990964076, + "recall": 0.986816580663952, + "f1-score": 0.9617464096253906, + "support": 181971.0 + }, + "1": { + "precision": 0.9269977481589678, + "recall": 0.7193322156367329, + "f1-score": 0.8100676762707583, + "support": 42349.0 + }, + "accuracy": 0.9363186519258203, + "macro avg": { + "precision": 0.9324581236276877, + "recall": 0.8530743981503424, + "f1-score": 0.8859070429480744, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9358567886762504, + "recall": 0.9363186519258203, + "f1-score": 0.9331112336320092, + "support": 224320.0 + }, + "cross-entropy": 0.14950830177513505, + "roc-auc": 0.9756752367295682, + "score": 0.9756752367295682 + }, + "val": { + "0": { + "precision": 0.9291661978170361, + "recall": 0.9865002090675586, + "f1-score": 0.9569752281616688, + "support": 16741.0 + }, + "1": { + "precision": 0.9043993231810491, + "recall": 0.6293788637032676, + "f1-score": 0.7422322513452526, + "support": 3397.0 + }, + "accuracy": 0.9262588141821432, + "macro avg": { + "precision": 0.9167827604990426, + "recall": 0.8079395363854132, + "f1-score": 0.8496037397534607, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9249883711640194, + "recall": 0.9262588141821432, + "f1-score": 0.9207510801705393, + "support": 20138.0 + }, + "cross-entropy": 0.1746148873886249, + "roc-auc": 0.9597570226838343, + "score": 0.9597570226838343 + }, + "test": { + "0": { + "precision": 0.9218071860547847, + "recall": 0.9854719707918156, + "f1-score": 0.9525770163958532, + "support": 13147.0 + }, + "1": { + "precision": 0.9147321428571429, + "recall": 0.6508894536213469, + "f1-score": 0.7605790645879733, + "support": 3148.0 + }, + "accuracy": 0.920834611844124, + "macro avg": { + "precision": 0.9182696644559638, + "recall": 0.8181807122065812, + "f1-score": 0.8565780404919132, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9204403719408739, + "recall": 0.920834611844124, + "f1-score": 0.9154852979367427, + "support": 16295.0 + }, + "cross-entropy": 0.17973653945792817, + "roc-auc": 0.9633062567165207, + "score": 0.9633062567165207 + } + }, + "time": "0:03:52.600341", + "diversity": { + "train": { + "mean_centered_corr": 0.0775546282529831, + "mean_pairwise_disagreement": 0.0356185432101606, + "member_std": 0.03550577908754349 + }, + "val": { + "mean_centered_corr": 0.08212897181510925, + "mean_pairwise_disagreement": 0.03171593825186856, + "member_std": 0.03616226091980934 + }, + "test": { + "mean_centered_corr": 0.10865671932697296, + "mean_pairwise_disagreement": 0.038570855892862445, + "member_std": 0.038954634219408035 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31, + 28, + 30, + 17, + 22, + 12, + 29, + 28, + 18, + 6 + ], + "head_selection_time": "0:00:13.698772" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.7-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.7-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.7-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.7-best-head-evaluation/0/report.json new file mode 100644 index 000000000..ba4f4903d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.7-best-head-evaluation/0/report.json @@ -0,0 +1,298 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.7051174640655518, + "min_features_per_member": 194.0, + "mean_features_per_member": 210.125, + "max_features_per_member": 226.0, + "min_members_per_feature": 15.0, + "mean_members_per_feature": 22.563758850097656, + "max_members_per_feature": 30.0, + "mean_pairwise_jaccard": 0.5455597077325217, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9507069533217122, + "recall": 0.971066818339186, + "f1-score": 0.9607790364806342, + "support": 181971.0 + }, + "1": { + "precision": 0.8630760428586289, + "recall": 0.7836548678835391, + "f1-score": 0.8214502295763666, + "support": 42349.0 + }, + "accuracy": 0.9356856276747504, + "macro avg": { + "precision": 0.9068914980901706, + "recall": 0.8773608431113625, + "f1-score": 0.8911146330285005, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9341632593702094, + "recall": 0.9356856276747504, + "f1-score": 0.9344753825773316, + "support": 224320.0 + }, + "cross-entropy": 0.14753096113026493, + "roc-auc": 0.975139322227928, + "score": 0.975139322227928 + }, + "val": { + "0": { + "precision": 0.9388616006934412, + "recall": 0.9704916074308584, + "f1-score": 0.9544146155201786, + "support": 16741.0 + }, + "1": { + "precision": 0.8256265442993294, + "recall": 0.6885487194583456, + "f1-score": 0.7508828250401284, + "support": 3397.0 + }, + "accuracy": 0.9229317707816069, + "macro avg": { + "precision": 0.8822440724963853, + "recall": 0.829520163444602, + "f1-score": 0.8526487202801535, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9197604244807687, + "recall": 0.9229317707816069, + "f1-score": 0.9200816384489338, + "support": 20138.0 + }, + "cross-entropy": 0.18416591007766794, + "roc-auc": 0.9540607823461205, + "score": 0.9540607823461205 + }, + "test": { + "0": { + "precision": 0.9347538574577516, + "recall": 0.9676732334372861, + "f1-score": 0.9509287289307471, + "support": 13147.0 + }, + "1": { + "precision": 0.8417132216014898, + "recall": 0.7179161372299873, + "f1-score": 0.7749014229384537, + "support": 3148.0 + }, + "accuracy": 0.9194231359312672, + "macro avg": { + "precision": 0.8882335395296207, + "recall": 0.8427946853336367, + "f1-score": 0.8629150759346004, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9167795143049124, + "recall": 0.9194231359312672, + "f1-score": 0.9169223491048043, + "support": 16295.0 + }, + "cross-entropy": 0.1854429639680667, + "roc-auc": 0.9604495578247303, + "score": 0.9604495578247303 + } + }, + "time": "0:03:07.457038", + "diversity": { + "train": { + "mean_centered_corr": 0.1248728483915329, + "mean_pairwise_disagreement": 0.057014991184828126, + "member_std": 0.05655066296458244 + }, + "val": { + "mean_centered_corr": 0.11596227437257767, + "mean_pairwise_disagreement": 0.055349840135324326, + "member_std": 0.06094019487500191 + }, + "test": { + "mean_centered_corr": 0.14900994300842285, + "mean_pairwise_disagreement": 0.06749510041671203, + "member_std": 0.06686553359031677 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 20 + ], + "head_selection_time": "0:00:01.696307" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.7-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.7-evaluation/0.toml new file mode 100644 index 000000000..137e9f011 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.7-evaluation/0.toml @@ -0,0 +1,47 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.7 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.7-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.7-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.7-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.7-evaluation/0/report.json new file mode 100644 index 000000000..c760f4271 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.7-evaluation/0/report.json @@ -0,0 +1,294 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.7051174640655518, + "min_features_per_member": 194.0, + "mean_features_per_member": 210.125, + "max_features_per_member": 226.0, + "min_members_per_feature": 15.0, + "mean_members_per_feature": 22.563758850097656, + "max_members_per_feature": 30.0, + "mean_pairwise_jaccard": 0.5455597077325217, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9454041901191536, + "recall": 0.9867176638035731, + "f1-score": 0.9656192355886355, + "support": 181971.0 + }, + "1": { + "precision": 0.9297322440910545, + "recall": 0.7551536045715365, + "f1-score": 0.8333984833085764, + "support": 42349.0 + }, + "accuracy": 0.9430010699001427, + "macro avg": { + "precision": 0.9375682171051041, + "recall": 0.8709356341875547, + "f1-score": 0.8995088594486059, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9424455094738968, + "recall": 0.9430010699001427, + "f1-score": 0.9406574995048791, + "support": 224320.0 + }, + "cross-entropy": 0.1482790588601011, + "roc-auc": 0.9807497384825534, + "score": 0.9807497384825534 + }, + "val": { + "0": { + "precision": 0.9299046974567191, + "recall": 0.9850068693626426, + "f1-score": 0.9566629924000696, + "support": 16741.0 + }, + "1": { + "precision": 0.8956340956340957, + "recall": 0.6340889019723285, + "f1-score": 0.7425025853154085, + "support": 3397.0 + }, + "accuracy": 0.9258118979044593, + "macro avg": { + "precision": 0.9127693965454073, + "recall": 0.8095478856674856, + "f1-score": 0.849582788857739, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9241237244508868, + "recall": 0.9258118979044593, + "f1-score": 0.9205371158052441, + "support": 20138.0 + }, + "cross-entropy": 0.1836966792126167, + "roc-auc": 0.9574455596570353, + "score": 0.9574455596570353 + }, + "test": { + "0": { + "precision": 0.9292490404808458, + "recall": 0.9760401612535179, + "f1-score": 0.9520700400652916, + "support": 13147.0 + }, + "1": { + "precision": 0.8732904263877715, + "recall": 0.6896442185514613, + "f1-score": 0.7706780262690806, + "support": 3148.0 + }, + "accuracy": 0.9207118748082234, + "macro avg": { + "precision": 0.9012697334343087, + "recall": 0.8328421899024896, + "f1-score": 0.8613740331671861, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9184385024529232, + "recall": 0.9207118748082234, + "f1-score": 0.9170272625611203, + "support": 16295.0 + }, + "cross-entropy": 0.1892419229433621, + "roc-auc": 0.9607387445394369, + "score": 0.9607387445394369 + } + }, + "time": "0:03:07.457038", + "diversity": { + "train": { + "mean_centered_corr": 0.1248728483915329, + "mean_pairwise_disagreement": 0.057014991184828126, + "member_std": 0.05655066296458244 + }, + "val": { + "mean_centered_corr": 0.11596227437257767, + "mean_pairwise_disagreement": 0.055349840135324326, + "member_std": 0.06094019487500191 + }, + "test": { + "mean_centered_corr": 0.14900994300842285, + "mean_pairwise_disagreement": 0.06749510041671203, + "member_std": 0.06686553359031677 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.7-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.7-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.7-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.7-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..476c62b54 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.7-greedy-heads-evaluation/0/report.json @@ -0,0 +1,305 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.7051174640655518, + "min_features_per_member": 194.0, + "mean_features_per_member": 210.125, + "max_features_per_member": 226.0, + "min_members_per_feature": 15.0, + "mean_members_per_feature": 22.563758850097656, + "max_members_per_feature": 30.0, + "mean_pairwise_jaccard": 0.5455597077325217, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9461931452336468, + "recall": 0.9852064339922295, + "f1-score": 0.9653057653219185, + "support": 181971.0 + }, + "1": { + "precision": 0.9227457957871779, + "recall": 0.7592623202436893, + "f1-score": 0.8330591359544012, + "support": 42349.0 + }, + "accuracy": 0.9425508202567761, + "macro avg": { + "precision": 0.9344694705104124, + "recall": 0.8722343771179595, + "f1-score": 0.8991824506381598, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9417665590990689, + "recall": 0.9425508202567761, + "f1-score": 0.9403391439458264, + "support": 224320.0 + }, + "cross-entropy": 0.14359261649483174, + "roc-auc": 0.9800293959977575, + "score": 0.9800293959977575 + }, + "val": { + "0": { + "precision": 0.9331481061465184, + "recall": 0.9830356609521534, + "f1-score": 0.9574424760741194, + "support": 16741.0 + }, + "1": { + "precision": 0.8864908073541167, + "recall": 0.6529290550485722, + "f1-score": 0.7519918630276318, + "support": 3397.0 + }, + "accuracy": 0.9273512761942596, + "macro avg": { + "precision": 0.9098194567503175, + "recall": 0.8179823580003629, + "f1-score": 0.8547171695508755, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9252776699563413, + "recall": 0.9273512761942596, + "f1-score": 0.9227858203725146, + "support": 20138.0 + }, + "cross-entropy": 0.17676896401301107, + "roc-auc": 0.9588480241238587, + "score": 0.9588480241238587 + }, + "test": { + "0": { + "precision": 0.929438982070561, + "recall": 0.9778656727770594, + "f1-score": 0.9530375477223025, + "support": 13147.0 + }, + "1": { + "precision": 0.881851400730816, + "recall": 0.6899618805590851, + "f1-score": 0.7741935483870968, + "support": 3148.0 + }, + "accuracy": 0.9222460877569807, + "macro avg": { + "precision": 0.9056451914006884, + "recall": 0.8339137766680722, + "f1-score": 0.8636155480546996, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9202456279093141, + "recall": 0.9222460877569807, + "f1-score": 0.9184870162766303, + "support": 16295.0 + }, + "cross-entropy": 0.18025747533530698, + "roc-auc": 0.9630162122394903, + "score": 0.9630162122394903 + } + }, + "time": "0:03:07.457038", + "diversity": { + "train": { + "mean_centered_corr": 0.1248728483915329, + "mean_pairwise_disagreement": 0.057014991184828126, + "member_std": 0.05655066296458244 + }, + "val": { + "mean_centered_corr": 0.11596227437257767, + "mean_pairwise_disagreement": 0.055349840135324326, + "member_std": 0.06094019487500191 + }, + "test": { + "mean_centered_corr": 0.14900994300842285, + "mean_pairwise_disagreement": 0.06749510041671203, + "member_std": 0.06686553359031677 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 20, + 28, + 22, + 4, + 16, + 13, + 10, + 26 + ], + "head_selection_time": "0:00:10.918451" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.8-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.8-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.8-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.8-best-head-evaluation/0/report.json new file mode 100644 index 000000000..e7eac6ed6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.8-best-head-evaluation/0/report.json @@ -0,0 +1,298 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.805578887462616, + "min_features_per_member": 224.0, + "mean_features_per_member": 240.0625, + "max_features_per_member": 251.0, + "min_members_per_feature": 19.0, + "mean_members_per_feature": 25.77852439880371, + "max_members_per_feature": 31.0, + "mean_pairwise_jaccard": 0.6740198042716817, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9487459340304847, + "recall": 0.9697259453429392, + "f1-score": 0.9591212231564873, + "support": 181971.0 + }, + "1": { + "precision": 0.8562557077625571, + "recall": 0.7748943304446386, + "f1-score": 0.8135458759947443, + "support": 42349.0 + }, + "accuracy": 0.9329440085592011, + "macro avg": { + "precision": 0.9025008208965208, + "recall": 0.872310137893789, + "f1-score": 0.8863335495756157, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9312848579239384, + "recall": 0.9329440085592011, + "f1-score": 0.9316382952991734, + "support": 224320.0 + }, + "cross-entropy": 0.15551409232715008, + "roc-auc": 0.9725736580353728, + "score": 0.9725736580353728 + }, + "val": { + "0": { + "precision": 0.9397257116514924, + "recall": 0.9741353563108536, + "f1-score": 0.9566212054553453, + "support": 16741.0 + }, + "1": { + "precision": 0.8444683908045977, + "recall": 0.6920812481601413, + "f1-score": 0.7607183303672544, + "support": 3397.0 + }, + "accuracy": 0.9265567583672659, + "macro avg": { + "precision": 0.8920970512280451, + "recall": 0.8331083022354975, + "f1-score": 0.8586697679112999, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9236571289264501, + "recall": 0.9265567583672659, + "f1-score": 0.9235751201105123, + "support": 20138.0 + }, + "cross-entropy": 0.18078572473789445, + "roc-auc": 0.9557633918282306, + "score": 0.9557633918282306 + }, + "test": { + "0": { + "precision": 0.9362878843743087, + "recall": 0.9657716589335971, + "f1-score": 0.9508012580500225, + "support": 13147.0 + }, + "1": { + "precision": 0.8354059985369422, + "recall": 0.7255400254129606, + "f1-score": 0.7766065963957838, + "support": 3148.0 + }, + "accuracy": 0.919361767413317, + "macro avg": { + "precision": 0.8858469414556254, + "recall": 0.8456558421732788, + "f1-score": 0.8637039272229031, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9167987050790629, + "recall": 0.919361767413317, + "f1-score": 0.9171489232916583, + "support": 16295.0 + }, + "cross-entropy": 0.1855701112620712, + "roc-auc": 0.9594055402651032, + "score": 0.9594055402651032 + } + }, + "time": "0:03:00.284704", + "diversity": { + "train": { + "mean_centered_corr": 0.10049896687269211, + "mean_pairwise_disagreement": 0.04618599113881092, + "member_std": 0.04431634396314621 + }, + "val": { + "mean_centered_corr": 0.10512824356555939, + "mean_pairwise_disagreement": 0.042368444026539456, + "member_std": 0.046922747045755386 + }, + "test": { + "mean_centered_corr": 0.1312195509672165, + "mean_pairwise_disagreement": 0.05293071791267854, + "member_std": 0.051259808242321014 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 17 + ], + "head_selection_time": "0:00:01.208354" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.8-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.8-evaluation/0.toml new file mode 100644 index 000000000..1173eceb6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.8-evaluation/0.toml @@ -0,0 +1,47 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.8 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.8-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.8-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.8-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.8-evaluation/0/report.json new file mode 100644 index 000000000..3d703a52e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.8-evaluation/0/report.json @@ -0,0 +1,294 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.805578887462616, + "min_features_per_member": 224.0, + "mean_features_per_member": 240.0625, + "max_features_per_member": 251.0, + "min_members_per_feature": 19.0, + "mean_members_per_feature": 25.77852439880371, + "max_members_per_feature": 31.0, + "mean_pairwise_jaccard": 0.6740198042716817, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.945464060302995, + "recall": 0.9829093646789873, + "f1-score": 0.963823154582122, + "support": 181971.0 + }, + "1": { + "precision": 0.9115019065505663, + "recall": 0.7563814966114902, + "f1-score": 0.8267282652178962, + "support": 42349.0 + }, + "accuracy": 0.940143544935806, + "macro avg": { + "precision": 0.9284829834267807, + "recall": 0.8696454306452388, + "f1-score": 0.895275709900009, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9390524017381697, + "recall": 0.940143544935806, + "f1-score": 0.9379412382586306, + "support": 224320.0 + }, + "cross-entropy": 0.14707160080340106, + "roc-auc": 0.9781088321813957, + "score": 0.9781088321813957 + }, + "val": { + "0": { + "precision": 0.9330230446134635, + "recall": 0.9819007227764172, + "f1-score": 0.9568380919118717, + "support": 16741.0 + }, + "1": { + "precision": 0.8797619047619047, + "recall": 0.652634677656756, + "f1-score": 0.749366232888288, + "support": 3397.0 + }, + "accuracy": 0.9263581289105174, + "macro avg": { + "precision": 0.9063924746876841, + "recall": 0.8172677002165866, + "f1-score": 0.8531021624000799, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9240386324535794, + "recall": 0.9263581289105174, + "f1-score": 0.9218404801776819, + "support": 20138.0 + }, + "cross-entropy": 0.17738371149581372, + "roc-auc": 0.9580661506671708, + "score": 0.9580661506671708 + }, + "test": { + "0": { + "precision": 0.930227503260397, + "recall": 0.9765726021145509, + "f1-score": 0.9528368399569558, + "support": 13147.0 + }, + "1": { + "precision": 0.8764540713999198, + "recall": 0.6940914866581956, + "f1-score": 0.7746853394788158, + "support": 3148.0 + }, + "accuracy": 0.9220006136851795, + "macro avg": { + "precision": 0.9033407873301584, + "recall": 0.8353320443863732, + "f1-score": 0.8637610897178858, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9198391164241416, + "recall": 0.9220006136851795, + "f1-score": 0.9184200910459288, + "support": 16295.0 + }, + "cross-entropy": 0.18143273966935003, + "roc-auc": 0.9619981087669689, + "score": 0.9619981087669689 + } + }, + "time": "0:03:00.284704", + "diversity": { + "train": { + "mean_centered_corr": 0.10049896687269211, + "mean_pairwise_disagreement": 0.04618599113881092, + "member_std": 0.04431634396314621 + }, + "val": { + "mean_centered_corr": 0.10512824356555939, + "mean_pairwise_disagreement": 0.042368444026539456, + "member_std": 0.046922747045755386 + }, + "test": { + "mean_centered_corr": 0.1312195509672165, + "mean_pairwise_disagreement": 0.05293071791267854, + "member_std": 0.051259808242321014 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.8-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.8-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.8-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.8-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..5c5e6a95f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.8-greedy-heads-evaluation/0/report.json @@ -0,0 +1,305 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.805578887462616, + "min_features_per_member": 224.0, + "mean_features_per_member": 240.0625, + "max_features_per_member": 251.0, + "min_members_per_feature": 19.0, + "mean_members_per_feature": 25.77852439880371, + "max_members_per_feature": 31.0, + "mean_pairwise_jaccard": 0.6740198042716817, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9423305588585018, + "recall": 0.984250237675234, + "f1-score": 0.962834341745579, + "support": 181971.0 + }, + "1": { + "precision": 0.9163309394523267, + "recall": 0.7411745259628326, + "f1-score": 0.8194979308904351, + "support": 42349.0 + }, + "accuracy": 0.9383603780313837, + "macro avg": { + "precision": 0.9293307491554142, + "recall": 0.8627123818190333, + "f1-score": 0.891166136318007, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9374221339198779, + "recall": 0.9383603780313837, + "f1-score": 0.9357740989526738, + "support": 224320.0 + }, + "cross-entropy": 0.14563506125296627, + "roc-auc": 0.9772221952124672, + "score": 0.9772221952124672 + }, + "val": { + "0": { + "precision": 0.9319107991849672, + "recall": 0.9835135296577265, + "f1-score": 0.9570170594902496, + "support": 16741.0 + }, + "1": { + "precision": 0.8882591093117409, + "recall": 0.6458639976449808, + "f1-score": 0.7479120504516789, + "support": 3397.0 + }, + "accuracy": 0.9265567583672659, + "macro avg": { + "precision": 0.910084954248354, + "recall": 0.8146887636513537, + "f1-score": 0.8524645549709642, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.924547367339732, + "recall": 0.9265567583672659, + "f1-score": 0.9217439581046092, + "support": 20138.0 + }, + "cross-entropy": 0.17444583477097872, + "roc-auc": 0.9587700557017027, + "score": 0.9587700557017027 + }, + "test": { + "0": { + "precision": 0.9287004262081918, + "recall": 0.9778656727770594, + "f1-score": 0.9526491293071508, + "support": 13147.0 + }, + "1": { + "precision": 0.8813213703099511, + "recall": 0.6864675984752223, + "f1-score": 0.7717857142857143, + "support": 3148.0 + }, + "accuracy": 0.9215710340595274, + "macro avg": { + "precision": 0.9050108982590714, + "recall": 0.8321666356261408, + "f1-score": 0.8622174217964326, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9195473566796456, + "recall": 0.9215710340595274, + "f1-score": 0.9177084707930372, + "support": 16295.0 + }, + "cross-entropy": 0.179030199475604, + "roc-auc": 0.9623207965369405, + "score": 0.9623207965369405 + } + }, + "time": "0:03:00.284704", + "diversity": { + "train": { + "mean_centered_corr": 0.10049896687269211, + "mean_pairwise_disagreement": 0.04618599113881092, + "member_std": 0.04431634396314621 + }, + "val": { + "mean_centered_corr": 0.10512824356555939, + "mean_pairwise_disagreement": 0.042368444026539456, + "member_std": 0.046922747045755386 + }, + "test": { + "mean_centered_corr": 0.1312195509672165, + "mean_pairwise_disagreement": 0.05293071791267854, + "member_std": 0.051259808242321014 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 17, + 12, + 30, + 28, + 15, + 4, + 13, + 22 + ], + "head_selection_time": "0:00:13.417596" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.9-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.9-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.9-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.9-best-head-evaluation/0/report.json new file mode 100644 index 000000000..753101f67 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.9-best-head-evaluation/0/report.json @@ -0,0 +1,298 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.901321291923523, + "min_features_per_member": 256.0, + "mean_features_per_member": 268.59375, + "max_features_per_member": 279.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.842281341552734, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8197222809700668, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9518098162802934, + "recall": 0.9742541393958378, + "f1-score": 0.9629012065730121, + "support": 181971.0 + }, + "1": { + "precision": 0.8768984182037942, + "recall": 0.7880469432572198, + "f1-score": 0.8301018568035121, + "support": 42349.0 + }, + "accuracy": 0.9391003922967189, + "macro avg": { + "precision": 0.9143541172420437, + "recall": 0.8811505413265288, + "f1-score": 0.8965015316882621, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9376674179335491, + "recall": 0.9391003922967189, + "f1-score": 0.9378302380308022, + "support": 224320.0 + }, + "cross-entropy": 0.1417006278725686, + "roc-auc": 0.9768342693042886, + "score": 0.9768342693042886 + }, + "val": { + "0": { + "precision": 0.9406342396660676, + "recall": 0.9691774684905322, + "f1-score": 0.9546925566343042, + "support": 16741.0 + }, + "1": { + "precision": 0.8213914849428868, + "recall": 0.6985575507801001, + "f1-score": 0.755011135857461, + "support": 3397.0 + }, + "accuracy": 0.9235276591518522, + "macro avg": { + "precision": 0.8810128623044773, + "recall": 0.8338675096353161, + "f1-score": 0.8548518462458826, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9205196484556869, + "recall": 0.9235276591518522, + "f1-score": 0.9210090832814918, + "support": 20138.0 + }, + "cross-entropy": 0.18275894278340477, + "roc-auc": 0.9560981250001208, + "score": 0.9560981250001208 + }, + "test": { + "0": { + "precision": 0.9386612939332886, + "recall": 0.9567962272761846, + "f1-score": 0.9476420069308422, + "support": 13147.0 + }, + "1": { + "precision": 0.8037318590186593, + "recall": 0.738881829733164, + "f1-score": 0.7699437272426349, + "support": 3148.0 + }, + "accuracy": 0.9146977600490949, + "macro avg": { + "precision": 0.8711965764759739, + "recall": 0.8478390285046743, + "f1-score": 0.8587928670867386, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9125945335091553, + "recall": 0.9146977600490949, + "f1-score": 0.9133128148806136, + "support": 16295.0 + }, + "cross-entropy": 0.1874554754465997, + "roc-auc": 0.9601610718172741, + "score": 0.9601610718172741 + } + }, + "time": "0:03:06.389407", + "diversity": { + "train": { + "mean_centered_corr": 0.08821944147348404, + "mean_pairwise_disagreement": 0.03961999131425153, + "member_std": 0.03912428021430969 + }, + "val": { + "mean_centered_corr": 0.09983313083648682, + "mean_pairwise_disagreement": 0.03728577252442021, + "member_std": 0.04294782876968384 + }, + "test": { + "mean_centered_corr": 0.11663155257701874, + "mean_pairwise_disagreement": 0.04569306832691603, + "member_std": 0.04552939534187317 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31 + ], + "head_selection_time": "0:00:01.042460" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.9-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.9-evaluation/0.toml new file mode 100644 index 000000000..fe5f765a4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.9-evaluation/0.toml @@ -0,0 +1,47 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.9-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.9-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.9-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.9-evaluation/0/report.json new file mode 100644 index 000000000..600b47205 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.9-evaluation/0/report.json @@ -0,0 +1,294 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.901321291923523, + "min_features_per_member": 256.0, + "mean_features_per_member": 268.59375, + "max_features_per_member": 279.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.842281341552734, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8197222809700668, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.946151494708437, + "recall": 0.9865253254639476, + "f1-score": 0.9659167032097862, + "support": 181971.0 + }, + "1": { + "precision": 0.9291001619245894, + "recall": 0.7587428274575551, + "f1-score": 0.8353242431726308, + "support": 42349.0 + }, + "accuracy": 0.9435226462196862, + "macro avg": { + "precision": 0.9376258283165132, + "recall": 0.8726340764607513, + "f1-score": 0.9006204731912084, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9429324019299814, + "recall": 0.9435226462196862, + "f1-score": 0.94126236971249, + "support": 224320.0 + }, + "cross-entropy": 0.13654273925686128, + "roc-auc": 0.980952103177431, + "score": 0.980952103177431 + }, + "val": { + "0": { + "precision": 0.9326650809831238, + "recall": 0.9837524640105131, + "f1-score": 0.9575278351114858, + "support": 16741.0 + }, + "1": { + "precision": 0.8903225806451613, + "recall": 0.6499852811304092, + "f1-score": 0.7514037774374681, + "support": 3397.0 + }, + "accuracy": 0.9274505909226338, + "macro avg": { + "precision": 0.9114938308141426, + "recall": 0.8168688725704611, + "f1-score": 0.854465806274477, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9255224911704286, + "recall": 0.9274505909226338, + "f1-score": 0.9227575786848973, + "support": 20138.0 + }, + "cross-entropy": 0.17552008158175839, + "roc-auc": 0.9580942942079151, + "score": 0.9580942942079151 + }, + "test": { + "0": { + "precision": 0.9282410752998989, + "recall": 0.9770289799954363, + "f1-score": 0.9520103761348897, + "support": 13147.0 + }, + "1": { + "precision": 0.877085877085877, + "recall": 0.684561626429479, + "f1-score": 0.7689562890276539, + "support": 3148.0 + }, + "accuracy": 0.9205277692543725, + "macro avg": { + "precision": 0.902663476192888, + "recall": 0.8307953032124576, + "f1-score": 0.8604833325812717, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9183585000327775, + "recall": 0.9205277692543725, + "f1-score": 0.9166465058548297, + "support": 16295.0 + }, + "cross-entropy": 0.1804777281050399, + "roc-auc": 0.9616652728230258, + "score": 0.9616652728230258 + } + }, + "time": "0:03:06.389407", + "diversity": { + "train": { + "mean_centered_corr": 0.08821944147348404, + "mean_pairwise_disagreement": 0.03961999131425153, + "member_std": 0.03912428021430969 + }, + "val": { + "mean_centered_corr": 0.09983313083648682, + "mean_pairwise_disagreement": 0.03728577252442021, + "member_std": 0.04294782876968384 + }, + "test": { + "mean_centered_corr": 0.11663155257701874, + "mean_pairwise_disagreement": 0.04569306832691603, + "member_std": 0.04552939534187317 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.9-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.9-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.9-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.9-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..1034e97f1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.9-greedy-heads-evaluation/0/report.json @@ -0,0 +1,307 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.901321291923523, + "min_features_per_member": 256.0, + "mean_features_per_member": 268.59375, + "max_features_per_member": 279.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.842281341552734, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8197222809700668, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.946386196568126, + "recall": 0.9862395656450753, + "f1-score": 0.9659019657970156, + "support": 181971.0 + }, + "1": { + "precision": 0.9278094908608661, + "recall": 0.7599234928805875, + "f1-score": 0.8355163237489452, + "support": 42349.0 + }, + "accuracy": 0.9435137303851641, + "macro avg": { + "precision": 0.937097843714496, + "recall": 0.8730815292628313, + "f1-score": 0.9007091447729805, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9428791311704943, + "recall": 0.9435137303851641, + "f1-score": 0.9412866771241655, + "support": 224320.0 + }, + "cross-entropy": 0.13649569775840315, + "roc-auc": 0.9805235149654821, + "score": 0.9805235149654821 + }, + "val": { + "0": { + "precision": 0.9322954995754317, + "recall": 0.9837524640105131, + "f1-score": 0.9573330233098878, + "support": 16741.0 + }, + "1": { + "precision": 0.8900121310149616, + "recall": 0.6479246393876951, + "f1-score": 0.7499148211243611, + "support": 3397.0 + }, + "accuracy": 0.927102989373324, + "macro avg": { + "precision": 0.9111538152951966, + "recall": 0.815838551699104, + "f1-score": 0.8536239222171245, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9251628844696655, + "recall": 0.927102989373324, + "f1-score": 0.9223444627366316, + "support": 20138.0 + }, + "cross-entropy": 0.17444519969608893, + "roc-auc": 0.9587921238951639, + "score": 0.9587921238951639 + }, + "test": { + "0": { + "precision": 0.9292688218702538, + "recall": 0.9773332319160265, + "f1-score": 0.9526951879587752, + "support": 13147.0 + }, + "1": { + "precision": 0.8792544570502431, + "recall": 0.6893265565438373, + "f1-score": 0.7727920227920227, + "support": 3148.0 + }, + "accuracy": 0.921693771095428, + "macro avg": { + "precision": 0.9042616394602485, + "recall": 0.8333298942299319, + "f1-score": 0.862743605375399, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9196066420326721, + "recall": 0.921693771095428, + "f1-score": 0.9179400382843391, + "support": 16295.0 + }, + "cross-entropy": 0.1794710390286192, + "roc-auc": 0.9622146031450255, + "score": 0.9622146031450255 + } + }, + "time": "0:03:06.389407", + "diversity": { + "train": { + "mean_centered_corr": 0.08821944147348404, + "mean_pairwise_disagreement": 0.03961999131425153, + "member_std": 0.03912428021430969 + }, + "val": { + "mean_centered_corr": 0.09983313083648682, + "mean_pairwise_disagreement": 0.03728577252442021, + "member_std": 0.04294782876968384 + }, + "test": { + "mean_centered_corr": 0.11663155257701874, + "mean_pairwise_disagreement": 0.04569306832691603, + "member_std": 0.04552939534187317 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31, + 7, + 23, + 22, + 15, + 5, + 2, + 30, + 7, + 28 + ], + "head_selection_time": "0:00:11.075623" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.95-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.95-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.95-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.95-best-head-evaluation/0/report.json new file mode 100644 index 000000000..a5901fec9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.95-best-head-evaluation/0/report.json @@ -0,0 +1,298 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9506081938743591, + "min_features_per_member": 275.0, + "mean_features_per_member": 283.28125, + "max_features_per_member": 292.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.419462203979492, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9056525652610176, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9539299644574761, + "recall": 0.9690170411768908, + "f1-score": 0.9614143176489831, + "support": 181971.0 + }, + "1": { + "precision": 0.8571609536115122, + "recall": 0.7989090651491181, + "f1-score": 0.8270105108775361, + "support": 42349.0 + }, + "accuracy": 0.9369026390870185, + "macro avg": { + "precision": 0.9055454590344942, + "recall": 0.8839630531630045, + "f1-score": 0.8942124142632596, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9356611037214039, + "recall": 0.9369026390870185, + "f1-score": 0.9360404463358412, + "support": 224320.0 + }, + "cross-entropy": 0.14460702857706767, + "roc-auc": 0.9762667563935794, + "score": 0.9762667563935794 + }, + "val": { + "0": { + "precision": 0.9437383395522388, + "recall": 0.9669075921390597, + "f1-score": 0.9551824860590682, + "support": 16741.0 + }, + "1": { + "precision": 0.8144675150703282, + "recall": 0.7159258168972623, + "f1-score": 0.7620241265862447, + "support": 3397.0 + }, + "accuracy": 0.9245704637997815, + "macro avg": { + "precision": 0.8791029273112836, + "recall": 0.8414167045181611, + "f1-score": 0.8586033063226565, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9219321527032445, + "recall": 0.9245704637997815, + "f1-score": 0.9225993622568445, + "support": 20138.0 + }, + "cross-entropy": 0.1811185402192146, + "roc-auc": 0.9563975490624737, + "score": 0.9563975490624737 + }, + "test": { + "0": { + "precision": 0.9422366337380288, + "recall": 0.9504069369437894, + "f1-score": 0.9463041502574977, + "support": 13147.0 + }, + "1": { + "precision": 0.7851021753460777, + "recall": 0.7566709021601017, + "f1-score": 0.7706243934001941, + "support": 3148.0 + }, + "accuracy": 0.9129794415464867, + "macro avg": { + "precision": 0.8636694045420532, + "recall": 0.8535389195519456, + "f1-score": 0.8584642718288459, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.911880127139817, + "recall": 0.9129794415464867, + "f1-score": 0.9123649127866912, + "support": 16295.0 + }, + "cross-entropy": 0.19078130335337062, + "roc-auc": 0.9593891219693567, + "score": 0.9593891219693567 + } + }, + "time": "0:03:07.006918", + "diversity": { + "train": { + "mean_centered_corr": 0.08914346992969513, + "mean_pairwise_disagreement": 0.03838632562640928, + "member_std": 0.037065718322992325 + }, + "val": { + "mean_centered_corr": 0.09447728097438812, + "mean_pairwise_disagreement": 0.0372919796949436, + "member_std": 0.04158519208431244 + }, + "test": { + "mean_centered_corr": 0.11072365194559097, + "mean_pairwise_disagreement": 0.045909714042502645, + "member_std": 0.04358496516942978 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31 + ], + "head_selection_time": "0:00:00.952109" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.95-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.95-evaluation/0.toml new file mode 100644 index 000000000..49f80d6e4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.95-evaluation/0.toml @@ -0,0 +1,47 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.95 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.95-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.95-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.95-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.95-evaluation/0/report.json new file mode 100644 index 000000000..24e827ce9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.95-evaluation/0/report.json @@ -0,0 +1,294 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9506081938743591, + "min_features_per_member": 275.0, + "mean_features_per_member": 283.28125, + "max_features_per_member": 292.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.419462203979492, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9056525652610176, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9496583046818625, + "recall": 0.9835852965582428, + "f1-score": 0.9663241038432363, + "support": 181971.0 + }, + "1": { + "precision": 0.9166759651863423, + "recall": 0.7759569293253678, + "f1-score": 0.8404670255892169, + "support": 42349.0 + }, + "accuracy": 0.9443874821683309, + "macro avg": { + "precision": 0.9331671349341024, + "recall": 0.8797711129418053, + "f1-score": 0.9033955647162266, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9434316236222344, + "recall": 0.9443874821683309, + "f1-score": 0.9425637552029926, + "support": 224320.0 + }, + "cross-entropy": 0.1349038769118999, + "roc-auc": 0.9810848735528083, + "score": 0.9810848735528083 + }, + "val": { + "0": { + "precision": 0.9357175450604609, + "recall": 0.979929514365928, + "f1-score": 0.9573133370291483, + "support": 16741.0 + }, + "1": { + "precision": 0.8710667689946278, + "recall": 0.6682366794230203, + "f1-score": 0.756288522405464, + "support": 3397.0 + }, + "accuracy": 0.9273512761942596, + "macro avg": { + "precision": 0.9033921570275443, + "recall": 0.8240830968944741, + "f1-score": 0.8568009297173061, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.924811859972784, + "recall": 0.9273512761942596, + "f1-score": 0.9234032518530308, + "support": 20138.0 + }, + "cross-entropy": 0.17405502211594576, + "roc-auc": 0.9584246137411132, + "score": 0.9584246137411132 + }, + "test": { + "0": { + "precision": 0.9328956965718453, + "recall": 0.9728455160873203, + "f1-score": 0.9524518747440146, + "support": 13147.0 + }, + "1": { + "precision": 0.8618955512572534, + "recall": 0.7077509529860229, + "f1-score": 0.7772544915402059, + "support": 3148.0 + }, + "accuracy": 0.9216324025774778, + "macro avg": { + "precision": 0.8973956239145493, + "recall": 0.8402982345366716, + "f1-score": 0.8648531831421102, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9191793137887625, + "recall": 0.9216324025774778, + "f1-score": 0.9186058261201674, + "support": 16295.0 + }, + "cross-entropy": 0.1782989047843598, + "roc-auc": 0.962318960200698, + "score": 0.962318960200698 + } + }, + "time": "0:03:07.006918", + "diversity": { + "train": { + "mean_centered_corr": 0.08914346992969513, + "mean_pairwise_disagreement": 0.03838632562640928, + "member_std": 0.037065718322992325 + }, + "val": { + "mean_centered_corr": 0.09447728097438812, + "mean_pairwise_disagreement": 0.0372919796949436, + "member_std": 0.04158519208431244 + }, + "test": { + "mean_centered_corr": 0.11072365194559097, + "mean_pairwise_disagreement": 0.045909714042502645, + "member_std": 0.04358496516942978 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.95-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.95-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.95-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.95-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..633179f9a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.95-greedy-heads-evaluation/0/report.json @@ -0,0 +1,304 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9506081938743591, + "min_features_per_member": 275.0, + "mean_features_per_member": 283.28125, + "max_features_per_member": 292.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.419462203979492, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9056525652610176, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9503279377423628, + "recall": 0.9817773161657627, + "f1-score": 0.9657966721086376, + "support": 181971.0 + }, + "1": { + "precision": 0.9087180334186693, + "recall": 0.779498925594465, + "f1-score": 0.8391631501347298, + "support": 42349.0 + }, + "accuracy": 0.943589514978602, + "macro avg": { + "precision": 0.929522985580516, + "recall": 0.8806381208801138, + "f1-score": 0.9024799111216837, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9424724730570735, + "recall": 0.943589514978602, + "f1-score": 0.941889739948897, + "support": 224320.0 + }, + "cross-entropy": 0.13479785159389124, + "roc-auc": 0.9805680231536485, + "score": 0.9805680231536485 + }, + "val": { + "0": { + "precision": 0.9373999542648068, + "recall": 0.9794516456603548, + "f1-score": 0.9579645371425233, + "support": 16741.0 + }, + "1": { + "precision": 0.8699924414210128, + "recall": 0.6776567559611422, + "f1-score": 0.7618732417673341, + "support": 3397.0 + }, + "accuracy": 0.9285430529347503, + "macro avg": { + "precision": 0.9036961978429099, + "recall": 0.8285542008107485, + "f1-score": 0.8599188894549287, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9260292460946623, + "recall": 0.9285430529347503, + "f1-score": 0.9248866679206782, + "support": 20138.0 + }, + "cross-entropy": 0.17265539939712965, + "roc-auc": 0.9592957095194116, + "score": 0.9592957095194116 + }, + "test": { + "0": { + "precision": 0.9348464815630968, + "recall": 0.9680535483380239, + "f1-score": 0.9511602705429543, + "support": 13147.0 + }, + "1": { + "precision": 0.8433420365535248, + "recall": 0.7182337992376112, + "f1-score": 0.7757762909589981, + "support": 3148.0 + }, + "accuracy": 0.919791347038969, + "macro avg": { + "precision": 0.8890942590583109, + "recall": 0.8431436737878175, + "f1-score": 0.8634682807509761, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9171689121927296, + "recall": 0.919791347038969, + "f1-score": 0.9172781737199843, + "support": 16295.0 + }, + "cross-entropy": 0.1790101347961027, + "roc-auc": 0.9620218603265258, + "score": 0.9620218603265258 + } + }, + "time": "0:03:07.006918", + "diversity": { + "train": { + "mean_centered_corr": 0.08914346992969513, + "mean_pairwise_disagreement": 0.03838632562640928, + "member_std": 0.037065718322992325 + }, + "val": { + "mean_centered_corr": 0.09447728097438812, + "mean_pairwise_disagreement": 0.0372919796949436, + "member_std": 0.04158519208431244 + }, + "test": { + "mean_centered_corr": 0.11072365194559097, + "mean_pairwise_disagreement": 0.045909714042502645, + "member_std": 0.04358496516942978 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31, + 5, + 28, + 20, + 9, + 6, + 23 + ], + "head_selection_time": "0:00:08.820400" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_cf_fisd-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_cf_fisd-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_cf_fisd-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_cf_fisd-best-head-evaluation/0/report.json new file mode 100644 index 000000000..2b8177a57 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_cf_fisd-best-head-evaluation/0/report.json @@ -0,0 +1,634 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1100, + "metrics": { + "train": { + "0": { + "precision": 0.935987075725548, + "recall": 0.9806232861280094, + "f1-score": 0.9577854114110891, + "support": 181971.0 + }, + "1": { + "precision": 0.8952808054408838, + "recall": 0.7118231835462466, + "f1-score": 0.7930807682188897, + "support": 42349.0 + }, + "accuracy": 0.9298769614835949, + "macro avg": { + "precision": 0.9156339405832159, + "recall": 0.846223234837128, + "f1-score": 0.8754330898149894, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9283022066087272, + "recall": 0.9298769614835949, + "f1-score": 0.9266910955473834, + "support": 224320.0 + }, + "cross-entropy": 0.16110536301253237, + "roc-auc": 0.9697341258683269, + "score": 0.9697341258683269 + }, + "val": { + "0": { + "precision": 0.9298739306618641, + "recall": 0.9869183441849352, + "f1-score": 0.9575473064997537, + "support": 16741.0 + }, + "1": { + "precision": 0.9075949367088607, + "recall": 0.6332057697968796, + "f1-score": 0.7459684411305705, + "support": 3397.0 + }, + "accuracy": 0.9272519614658854, + "macro avg": { + "precision": 0.9187344336853625, + "recall": 0.8100620569909074, + "f1-score": 0.8517578738151621, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9261157748142947, + "recall": 0.9272519614658854, + "f1-score": 0.9218569000214979, + "support": 20138.0 + }, + "cross-entropy": 0.17873116424917943, + "roc-auc": 0.9580215219221477, + "score": 0.9580215219221477 + }, + "test": { + "0": { + "precision": 0.9240352258895969, + "recall": 0.9816688217844375, + "f1-score": 0.9519805266651914, + "support": 13147.0 + }, + "1": { + "precision": 0.8964776632302406, + "recall": 0.6629606099110547, + "f1-score": 0.7622352081811541, + "support": 3148.0 + }, + "accuracy": 0.9200981896287205, + "macro avg": { + "precision": 0.9102564445599187, + "recall": 0.8223147158477461, + "f1-score": 0.8571078674231727, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9187114328701643, + "recall": 0.9200981896287205, + "f1-score": 0.9153239901455382, + "support": 16295.0 + }, + "cross-entropy": 0.18401642120758016, + "roc-auc": 0.9612981867919294, + "score": 0.9612981867919294 + } + }, + "time": "0:06:46.520241", + "diversity": { + "train": { + "mean_centered_corr": 0.018766077235341072, + "mean_pairwise_disagreement": 0.023167984748170816, + "member_std": 0.021088920533657074 + }, + "val": { + "mean_centered_corr": 0.017404615879058838, + "mean_pairwise_disagreement": 0.01965370395881322, + "member_std": 0.021817108616232872 + }, + "test": { + "mean_centered_corr": 0.028502995148301125, + "mean_pairwise_disagreement": 0.024374065862277167, + "member_std": 0.022561587393283844 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 5 + ], + "head_selection_time": "0:00:01.486676" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_cf_fisd-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_cf_fisd-evaluation/0.toml new file mode 100644 index 000000000..8d11e7472 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_cf_fisd-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "homesite-insurance" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_cf_fisd-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_cf_fisd-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_cf_fisd-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_cf_fisd-evaluation/0/report.json new file mode 100644 index 000000000..dafc0fcd1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_cf_fisd-evaluation/0/report.json @@ -0,0 +1,630 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1100, + "metrics": { + "train": { + "0": { + "precision": 0.9356324122034465, + "recall": 0.9837171857054146, + "f1-score": 0.9590724737070513, + "support": 181971.0 + }, + "1": { + "precision": 0.9102039579355699, + "recall": 0.7092021063071147, + "f1-score": 0.7972287845406525, + "support": 42349.0 + }, + "accuracy": 0.9318919400855921, + "macro avg": { + "precision": 0.9229181850695082, + "recall": 0.8464596460062646, + "f1-score": 0.8781506291238519, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.930831816582056, + "recall": 0.9318919400855921, + "f1-score": 0.9285182725992239, + "support": 224320.0 + }, + "cross-entropy": 0.15573626126466394, + "roc-auc": 0.9717679542404869, + "score": 0.9717679542404869 + }, + "val": { + "0": { + "precision": 0.9283587121424959, + "recall": 0.9869183441849352, + "f1-score": 0.956743297237825, + "support": 16741.0 + }, + "1": { + "precision": 0.9064502349423323, + "recall": 0.6246688254342067, + "f1-score": 0.7396305332868596, + "support": 3397.0 + }, + "accuracy": 0.9258118979044593, + "macro avg": { + "precision": 0.9174044735424141, + "recall": 0.805793584809571, + "f1-score": 0.8481869152623422, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9246630573084033, + "recall": 0.9258118979044593, + "f1-score": 0.9201193991773706, + "support": 20138.0 + }, + "cross-entropy": 0.1749156826554181, + "roc-auc": 0.9584762673811861, + "score": 0.9584762673811861 + }, + "test": { + "0": { + "precision": 0.9240280160091481, + "recall": 0.9834182703278315, + "f1-score": 0.9527985555842146, + "support": 13147.0 + }, + "1": { + "precision": 0.9053408597481546, + "recall": 0.6623252858958069, + "f1-score": 0.7649972482113374, + "support": 3148.0 + }, + "accuracy": 0.9213869285056766, + "macro avg": { + "precision": 0.9146844378786514, + "recall": 0.8228717781118191, + "f1-score": 0.858897901897776, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9204178798993226, + "recall": 0.9213869285056766, + "f1-score": 0.9165175788668278, + "support": 16295.0 + }, + "cross-entropy": 0.17872381508706203, + "roc-auc": 0.9621948866927381, + "score": 0.9621948866927381 + } + }, + "time": "0:06:46.520241", + "diversity": { + "train": { + "mean_centered_corr": 0.018766077235341072, + "mean_pairwise_disagreement": 0.023167984748170816, + "member_std": 0.021088920533657074 + }, + "val": { + "mean_centered_corr": 0.017404615879058838, + "mean_pairwise_disagreement": 0.01965370395881322, + "member_std": 0.021817108616232872 + }, + "test": { + "mean_centered_corr": 0.028502995148301125, + "mean_pairwise_disagreement": 0.024374065862277167, + "member_std": 0.022561587393283844 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_cf_fisd-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_cf_fisd-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_cf_fisd-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_cf_fisd-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..cb053e547 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_cf_fisd-greedy-heads-evaluation/0/report.json @@ -0,0 +1,637 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1100, + "metrics": { + "train": { + "0": { + "precision": 0.9380125992360596, + "recall": 0.981095888905375, + "f1-score": 0.9590706419554123, + "support": 181971.0 + }, + "1": { + "precision": 0.8987967403136125, + "recall": 0.7214101867812699, + "f1-score": 0.800392978779146, + "support": 42349.0 + }, + "accuracy": 0.9320702567760343, + "macro avg": { + "precision": 0.918404669774836, + "recall": 0.8512530378433225, + "f1-score": 0.8797318103672791, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9306091024033799, + "recall": 0.9320702567760343, + "f1-score": 0.9291141496326069, + "support": 224320.0 + }, + "cross-entropy": 0.15562722826354536, + "roc-auc": 0.97165868278649, + "score": 0.97165868278649 + }, + "val": { + "0": { + "precision": 0.9312214128409283, + "recall": 0.9850666029508393, + "f1-score": 0.9573875181422351, + "support": 16741.0 + }, + "1": { + "precision": 0.8970769864141622, + "recall": 0.6414483367677363, + "f1-score": 0.7480260899416409, + "support": 3397.0 + }, + "accuracy": 0.927102989373324, + "macro avg": { + "precision": 0.9141491996275453, + "recall": 0.8132574698592878, + "f1-score": 0.852706804041938, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9254617238662671, + "recall": 0.927102989373324, + "f1-score": 0.9220711624168692, + "support": 20138.0 + }, + "cross-entropy": 0.1746060587875626, + "roc-auc": 0.9589731006657612, + "score": 0.9589731006657612 + }, + "test": { + "0": { + "precision": 0.9252939489532549, + "recall": 0.9816688217844375, + "f1-score": 0.9526480900535154, + "support": 13147.0 + }, + "1": { + "precision": 0.8973157221985514, + "recall": 0.6689961880559085, + "f1-score": 0.7665150136487716, + "support": 3148.0 + }, + "accuracy": 0.921264191469776, + "macro avg": { + "precision": 0.9113048355759031, + "recall": 0.825332504920173, + "f1-score": 0.8595815518511435, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9198888886388145, + "recall": 0.921264191469776, + "f1-score": 0.9166893956980607, + "support": 16295.0 + }, + "cross-entropy": 0.1790016934189047, + "roc-auc": 0.9623105275513741, + "score": 0.9623105275513741 + } + }, + "time": "0:06:46.520241", + "diversity": { + "train": { + "mean_centered_corr": 0.018766077235341072, + "mean_pairwise_disagreement": 0.023167984748170816, + "member_std": 0.021088920533657074 + }, + "val": { + "mean_centered_corr": 0.017404615879058838, + "mean_pairwise_disagreement": 0.01965370395881322, + "member_std": 0.021817108616232872 + }, + "test": { + "mean_centered_corr": 0.028502995148301125, + "mean_pairwise_disagreement": 0.024374065862277167, + "member_std": 0.022561587393283844 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 5, + 15, + 21, + 24 + ], + "head_selection_time": "0:00:08.828635" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_esam-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_esam-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_esam-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_esam-best-head-evaluation/0/report.json new file mode 100644 index 000000000..ea0311c4d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_esam-best-head-evaluation/0/report.json @@ -0,0 +1,285 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_esam-evaluation/_esam_diagnostics/rla_esam-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9441741289589772, + "recall": 0.9821235251770887, + "f1-score": 0.9627750123903721, + "support": 181971.0 + }, + "1": { + "precision": 0.9071499928642786, + "recall": 0.7504781694963282, + "f1-score": 0.8214101106171818, + "support": 42349.0 + }, + "accuracy": 0.9383915834522111, + "macro avg": { + "precision": 0.925662060911628, + "recall": 0.8663008473367084, + "f1-score": 0.8920925615037769, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9371844038364985, + "recall": 0.9383915834522111, + "f1-score": 0.9360869675205753, + "support": 224320.0 + }, + "cross-entropy": 0.14284985705983097, + "roc-auc": 0.9763333421966126, + "score": 0.9763333421966126 + }, + "val": { + "0": { + "precision": 0.9350575367437621, + "recall": 0.9804671166596978, + "f1-score": 0.9572240851436069, + "support": 16741.0 + }, + "1": { + "precision": 0.8734520123839009, + "recall": 0.6644097733294083, + "f1-score": 0.7547232904196622, + "support": 3397.0 + }, + "accuracy": 0.9271526467375112, + "macro avg": { + "precision": 0.9042547745638315, + "recall": 0.822438444994553, + "f1-score": 0.8559736877816346, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9246655431867828, + "recall": 0.9271526467375112, + "f1-score": 0.92306502269067, + "support": 20138.0 + }, + "cross-entropy": 0.18123454105665865, + "roc-auc": 0.9561241320583908, + "score": 0.9561241320583908 + }, + "test": { + "0": { + "precision": 0.9287627597191052, + "recall": 0.9758119723130753, + "f1-score": 0.9517062314540059, + "support": 13147.0 + }, + "1": { + "precision": 0.8718775181305399, + "recall": 0.6874205844980941, + "f1-score": 0.7687388987566608, + "support": 3148.0 + }, + "accuracy": 0.9200981896287205, + "macro avg": { + "precision": 0.9003201389248225, + "recall": 0.8316162784055847, + "f1-score": 0.8602225651053333, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9177732082910105, + "recall": 0.9200981896287205, + "f1-score": 0.9163591210930828, + "support": 16295.0 + }, + "cross-entropy": 0.1861704512569149, + "roc-auc": 0.960971584726283, + "score": 0.960971584726283 + } + }, + "time": "0:12:26.246333", + "diversity": { + "train": { + "mean_centered_corr": 0.055896881967782974, + "mean_pairwise_disagreement": 0.0307871046114997, + "member_std": 0.02823685109615326 + }, + "val": { + "mean_centered_corr": 0.06491804867982864, + "mean_pairwise_disagreement": 0.028161331970692545, + "member_std": 0.02979666367173195 + }, + "test": { + "mean_centered_corr": 0.08032963424921036, + "mean_pairwise_disagreement": 0.035900211820368406, + "member_std": 0.03238997608423233 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 12 + ], + "head_selection_time": "0:00:00.738599" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_esam-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_esam-evaluation/0.toml new file mode 100644 index 000000000..b0b8fc3b1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_esam-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_esam-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_esam-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_esam-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_esam-evaluation/0/report.json new file mode 100644 index 000000000..4c2a205bb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_esam-evaluation/0/report.json @@ -0,0 +1,281 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_esam-evaluation/_esam_diagnostics/rla_esam-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9463842112495904, + "recall": 0.9839699732374939, + "f1-score": 0.9648111776576169, + "support": 181971.0 + }, + "1": { + "precision": 0.9169466431296623, + "recall": 0.7604665989751824, + "f1-score": 0.8314078816589433, + "support": 42349.0 + }, + "accuracy": 0.9417751426533524, + "macro avg": { + "precision": 0.9316654271896263, + "recall": 0.8722182861063381, + "f1-score": 0.8981095296582802, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9408267416868638, + "recall": 0.9417751426533524, + "f1-score": 0.9396261911105064, + "support": 224320.0 + }, + "cross-entropy": 0.1368376901407188, + "roc-auc": 0.9794268475505917, + "score": 0.9794268475505917 + }, + "val": { + "0": { + "precision": 0.9354489516864175, + "recall": 0.980765784600681, + "f1-score": 0.9575715160528387, + "support": 16741.0 + }, + "1": { + "precision": 0.8754833720030936, + "recall": 0.6664704150721225, + "f1-score": 0.7568109643991309, + "support": 3397.0 + }, + "accuracy": 0.9277485351077565, + "macro avg": { + "precision": 0.9054661618447555, + "recall": 0.8236180998364018, + "f1-score": 0.8571912402259848, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9253335939456164, + "recall": 0.9277485351077565, + "f1-score": 0.9237060083575539, + "support": 20138.0 + }, + "cross-entropy": 0.17396784432743392, + "roc-auc": 0.9581143402163179, + "score": 0.9581143402163179 + }, + "test": { + "0": { + "precision": 0.931127077436679, + "recall": 0.9758880352932228, + "f1-score": 0.9529822476416846, + "support": 13147.0 + }, + "1": { + "precision": 0.874006359300477, + "recall": 0.6985387547649301, + "f1-score": 0.7764830508474576, + "support": 3148.0 + }, + "accuracy": 0.9223074562749309, + "macro avg": { + "precision": 0.9025667183685779, + "recall": 0.8372133950290764, + "f1-score": 0.864732649244571, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9200920347430452, + "recall": 0.9223074562749309, + "f1-score": 0.9188847041309005, + "support": 16295.0 + }, + "cross-entropy": 0.17786783844384593, + "roc-auc": 0.962225790298713, + "score": 0.962225790298713 + } + }, + "time": "0:12:26.246333", + "diversity": { + "train": { + "mean_centered_corr": 0.055896881967782974, + "mean_pairwise_disagreement": 0.0307871046114997, + "member_std": 0.02823685109615326 + }, + "val": { + "mean_centered_corr": 0.06491804867982864, + "mean_pairwise_disagreement": 0.028161331970692545, + "member_std": 0.02979666367173195 + }, + "test": { + "mean_centered_corr": 0.08032963424921036, + "mean_pairwise_disagreement": 0.035900211820368406, + "member_std": 0.03238997608423233 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_esam-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_esam-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_esam-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_esam-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..fd4df6b14 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_esam-greedy-heads-evaluation/0/report.json @@ -0,0 +1,295 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_esam-evaluation/_esam_diagnostics/rla_esam-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.943986898505545, + "recall": 0.9851294986563793, + "f1-score": 0.9641194708946333, + "support": 181971.0 + }, + "1": { + "precision": 0.9213783485385554, + "recall": 0.7488252379040827, + "f1-score": 0.8261883361340159, + "support": 42349.0 + }, + "accuracy": 0.9405180099857346, + "macro avg": { + "precision": 0.9326826235220502, + "recall": 0.8669773682802311, + "f1-score": 0.8951539035143246, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9397186679306875, + "recall": 0.9405180099857346, + "f1-score": 0.9380796811925229, + "support": 224320.0 + }, + "cross-entropy": 0.13886662867003355, + "roc-auc": 0.9784331962449399, + "score": 0.9784331962449399 + }, + "val": { + "0": { + "precision": 0.9338176864570326, + "recall": 0.9827369930111702, + "f1-score": 0.9576530166768532, + "support": 16741.0 + }, + "1": { + "precision": 0.8853174603174603, + "recall": 0.6567559611421843, + "f1-score": 0.7540983606557377, + "support": 3397.0 + }, + "accuracy": 0.9277485351077565, + "macro avg": { + "precision": 0.9095675733872464, + "recall": 0.8197464770766772, + "f1-score": 0.8558756886662955, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9256363741024727, + "recall": 0.9277485351077565, + "f1-score": 0.9233161825074357, + "support": 20138.0 + }, + "cross-entropy": 0.17316436914109928, + "roc-auc": 0.9589296236870106, + "score": 0.9589296236870106 + }, + "test": { + "0": { + "precision": 0.9291702804278693, + "recall": 0.9778656727770594, + "f1-score": 0.9528962680206056, + "support": 13147.0 + }, + "1": { + "precision": 0.8816592110614071, + "recall": 0.6886912325285895, + "f1-score": 0.773319065453897, + "support": 3148.0 + }, + "accuracy": 0.9220006136851795, + "macro avg": { + "precision": 0.9054147457446382, + "recall": 0.8332784526528245, + "f1-score": 0.8631076667372513, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.919991707468948, + "recall": 0.9220006136851795, + "f1-score": 0.9182040904397527, + "support": 16295.0 + }, + "cross-entropy": 0.17763494877994662, + "roc-auc": 0.9624745655349262, + "score": 0.9624745655349262 + } + }, + "time": "0:12:26.246333", + "diversity": { + "train": { + "mean_centered_corr": 0.055896881967782974, + "mean_pairwise_disagreement": 0.0307871046114997, + "member_std": 0.02823685109615326 + }, + "val": { + "mean_centered_corr": 0.06491804867982864, + "mean_pairwise_disagreement": 0.028161331970692545, + "member_std": 0.02979666367173195 + }, + "test": { + "mean_centered_corr": 0.08032963424921036, + "mean_pairwise_disagreement": 0.035900211820368406, + "member_std": 0.03238997608423233 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 12, + 31, + 24, + 17, + 15, + 29, + 21, + 13, + 24, + 17, + 5 + ], + "head_selection_time": "0:00:11.343819" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_mfb-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_mfb-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_mfb-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_mfb-best-head-evaluation/0/report.json new file mode 100644 index 000000000..f4f8dc42f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_mfb-best-head-evaluation/0/report.json @@ -0,0 +1,311 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.901321291923523, + "min_features_per_member": 256.0, + "mean_features_per_member": 268.59375, + "max_features_per_member": 279.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.842281341552734, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8197222809700668, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9484513038850453, + "recall": 0.9793538530864808, + "f1-score": 0.9636548958961367, + "support": 181971.0 + }, + "1": { + "precision": 0.8968423942888523, + "recall": 0.7712814942501593, + "f1-score": 0.8293364140715256, + "support": 42349.0 + }, + "accuracy": 0.940072218259629, + "macro avg": { + "precision": 0.9226468490869488, + "recall": 0.8753176736683201, + "f1-score": 0.8964956549838312, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9387081436118231, + "recall": 0.940072218259629, + "f1-score": 0.9382971329379053, + "support": 224320.0 + }, + "cross-entropy": 0.1391744611707364, + "roc-auc": 0.9777392813930507, + "score": 0.9777392813930507 + }, + "val": { + "0": { + "precision": 0.9401502021952629, + "recall": 0.9721044143121678, + "f1-score": 0.9558603271563243, + "support": 16741.0 + }, + "1": { + "precision": 0.8348656294200849, + "recall": 0.6950250220783044, + "f1-score": 0.7585542168674699, + "support": 3397.0 + }, + "accuracy": 0.9253649816267753, + "macro avg": { + "precision": 0.887507915807674, + "recall": 0.8335647181952361, + "f1-score": 0.8572072720118971, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9223901617882077, + "recall": 0.9253649816267753, + "f1-score": 0.9225775355856004, + "support": 20138.0 + }, + "cross-entropy": 0.1813000791881102, + "roc-auc": 0.9554584990741118, + "score": 0.9554584990741118 + }, + "test": { + "0": { + "precision": 0.9373008226487809, + "recall": 0.961968509926219, + "f1-score": 0.9494744744744745, + "support": 13147.0 + }, + "1": { + "precision": 0.8215560314061384, + "recall": 0.7312579415501906, + "f1-score": 0.773781512605042, + "support": 3148.0 + }, + "accuracy": 0.9173979748389076, + "macro avg": { + "precision": 0.8794284270274597, + "recall": 0.8466132257382049, + "f1-score": 0.8616279935397583, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9149403069794444, + "recall": 0.9173979748389076, + "f1-score": 0.9155326859525368, + "support": 16295.0 + }, + "cross-entropy": 0.18597181508732155, + "roc-auc": 0.9598635611836792, + "score": 0.9598635611836792 + } + }, + "time": "0:07:36.712536", + "diversity": { + "train": { + "mean_centered_corr": 0.08504261821508408, + "mean_pairwise_disagreement": 0.037604994736781555, + "member_std": 0.03812075033783913 + }, + "val": { + "mean_centered_corr": 0.09174208343029022, + "mean_pairwise_disagreement": 0.03609920179791695, + "member_std": 0.04196608066558838 + }, + "test": { + "mean_centered_corr": 0.10859175771474838, + "mean_pairwise_disagreement": 0.04310061962406834, + "member_std": 0.044081997126340866 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 31 + ], + "head_selection_time": "0:00:01.202505" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_mfb-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_mfb-evaluation/0.toml new file mode 100644 index 000000000..9df7aee00 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_mfb-evaluation/0.toml @@ -0,0 +1,54 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_mfb-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_mfb-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_mfb-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_mfb-evaluation/0/report.json new file mode 100644 index 000000000..7f868dc23 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_mfb-evaluation/0/report.json @@ -0,0 +1,307 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.901321291923523, + "min_features_per_member": 256.0, + "mean_features_per_member": 268.59375, + "max_features_per_member": 279.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.842281341552734, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8197222809700668, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9459871673503734, + "recall": 0.9884542042413351, + "f1-score": 0.9667545436711089, + "support": 181971.0 + }, + "1": { + "precision": 0.9385313048566413, + "recall": 0.7574913221091407, + "f1-score": 0.8383488612160096, + "support": 42349.0 + }, + "accuracy": 0.9448511055634807, + "macro avg": { + "precision": 0.9422592361035074, + "recall": 0.872972763175238, + "f1-score": 0.9025517024435592, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9445795874611657, + "recall": 0.9448511055634807, + "f1-score": 0.942513048279298, + "support": 224320.0 + }, + "cross-entropy": 0.13350397534467387, + "roc-auc": 0.9820799272842926, + "score": 0.9820799272842926 + }, + "val": { + "0": { + "precision": 0.931040325313453, + "recall": 0.9847082014216594, + "f1-score": 0.9571225360699045, + "support": 16741.0 + }, + "1": { + "precision": 0.8947368421052632, + "recall": 0.6405652045922873, + "f1-score": 0.7466117687424945, + "support": 3397.0 + }, + "accuracy": 0.9266560730956401, + "macro avg": { + "precision": 0.9128885837093581, + "recall": 0.8126367030069733, + "f1-score": 0.8518671524061995, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.924916433543753, + "recall": 0.9266560730956401, + "f1-score": 0.9216123028485712, + "support": 20138.0 + }, + "cross-entropy": 0.1756831254542049, + "roc-auc": 0.9581138918187615, + "score": 0.9581138918187615 + }, + "test": { + "0": { + "precision": 0.9275205538727823, + "recall": 0.9782459876777972, + "f1-score": 0.9522081960537518, + "support": 13147.0 + }, + "1": { + "precision": 0.8822560724578016, + "recall": 0.6807496823379924, + "f1-score": 0.768513537744307, + "support": 3148.0 + }, + "accuracy": 0.9207732433261737, + "macro avg": { + "precision": 0.904888313165292, + "recall": 0.8294978350078948, + "f1-score": 0.8603608668990295, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9187759949593512, + "recall": 0.9207732433261737, + "f1-score": 0.9167205750437406, + "support": 16295.0 + }, + "cross-entropy": 0.18070868511872268, + "roc-auc": 0.9616453389098677, + "score": 0.9616453389098677 + } + }, + "time": "0:07:36.712536", + "diversity": { + "train": { + "mean_centered_corr": 0.08504261821508408, + "mean_pairwise_disagreement": 0.037604994736781555, + "member_std": 0.03812075033783913 + }, + "val": { + "mean_centered_corr": 0.09174208343029022, + "mean_pairwise_disagreement": 0.03609920179791695, + "member_std": 0.04196608066558838 + }, + "test": { + "mean_centered_corr": 0.10859175771474838, + "mean_pairwise_disagreement": 0.04310061962406834, + "member_std": 0.044081997126340866 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_mfb-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_mfb-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_mfb-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_mfb-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..47304e5df --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_mfb-greedy-heads-evaluation/0/report.json @@ -0,0 +1,320 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.901321291923523, + "min_features_per_member": 256.0, + "mean_features_per_member": 268.59375, + "max_features_per_member": 279.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.842281341552734, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8197222809700668, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9458629588665929, + "recall": 0.986920992905463, + "f1-score": 0.9659558789912893, + "support": 181971.0 + }, + "1": { + "precision": 0.9309143686502177, + "recall": 0.7572788023329948, + "f1-score": 0.8351671245719345, + "support": 42349.0 + }, + "accuracy": 0.9435672253922968, + "macro avg": { + "precision": 0.9383886637584054, + "recall": 0.8720998976192289, + "f1-score": 0.900561501781612, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9430408393628782, + "recall": 0.9435672253922968, + "f1-score": 0.9412644874038016, + "support": 224320.0 + }, + "cross-entropy": 0.13696330439392607, + "roc-auc": 0.9808254468336747, + "score": 0.9808254468336747 + }, + "val": { + "0": { + "precision": 0.9327445652173914, + "recall": 0.9841705991278896, + "f1-score": 0.957767766313036, + "support": 16741.0 + }, + "1": { + "precision": 0.8928860145513339, + "recall": 0.6502796585222255, + "f1-score": 0.7525123488332481, + "support": 3397.0 + }, + "accuracy": 0.9278478498361307, + "macro avg": { + "precision": 0.9128152898843627, + "recall": 0.8172251288250576, + "f1-score": 0.855140057573142, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9260209831033485, + "recall": 0.9278478498361307, + "f1-score": 0.9231440373837064, + "support": 20138.0 + }, + "cross-entropy": 0.1752850415945319, + "roc-auc": 0.9589065795694565, + "score": 0.9589065795694565 + }, + "test": { + "0": { + "precision": 0.9281277087546952, + "recall": 0.9773332319160265, + "f1-score": 0.9520951428253862, + "support": 13147.0 + }, + "1": { + "precision": 0.8784169726642187, + "recall": 0.6839263024142312, + "f1-score": 0.769065904625826, + "support": 3148.0 + }, + "accuracy": 0.9206505062902731, + "macro avg": { + "precision": 0.903272340709457, + "recall": 0.8306297671651288, + "f1-score": 0.8605805237256061, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.91852418637281, + "recall": 0.9206505062902731, + "f1-score": 0.9167360730584506, + "support": 16295.0 + }, + "cross-entropy": 0.18154574753822755, + "roc-auc": 0.961628401124263, + "score": 0.961628401124263 + } + }, + "time": "0:07:36.712536", + "diversity": { + "train": { + "mean_centered_corr": 0.08504261821508408, + "mean_pairwise_disagreement": 0.037604994736781555, + "member_std": 0.03812075033783913 + }, + "val": { + "mean_centered_corr": 0.09174208343029022, + "mean_pairwise_disagreement": 0.03609920179791695, + "member_std": 0.04196608066558838 + }, + "test": { + "mean_centered_corr": 0.10859175771474838, + "mean_pairwise_disagreement": 0.04310061962406834, + "member_std": 0.044081997126340866 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 31, + 4, + 28, + 22, + 29, + 2, + 30, + 5, + 26, + 6 + ], + "head_selection_time": "0:00:11.216798" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.0001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.0001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.0001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.0001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..7fa0e6c12 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.0001-best-head-evaluation/0/report.json @@ -0,0 +1,275 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3057696, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9463394409095983, + "recall": 0.978353693720428, + "f1-score": 0.9620803138628147, + "support": 181971.0 + }, + "1": { + "precision": 0.8911668002099854, + "recall": 0.7616236510897542, + "f1-score": 0.8213185302131344, + "support": 42349.0 + }, + "accuracy": 0.9374375891583452, + "macro avg": { + "precision": 0.9187531205597919, + "recall": 0.8699886724050911, + "f1-score": 0.8916994220379746, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9359234897639674, + "recall": 0.9374375891583452, + "f1-score": 0.9355061306612262, + "support": 224320.0 + }, + "cross-entropy": 0.14418027001253625, + "roc-auc": 0.9761245839581788, + "score": 0.9761245839581788 + }, + "val": { + "0": { + "precision": 0.9370957586858222, + "recall": 0.9779583059554388, + "f1-score": 0.9570910791535134, + "support": 16741.0 + }, + "1": { + "precision": 0.8616422947131609, + "recall": 0.6764792463938769, + "f1-score": 0.7579155672823219, + "support": 3397.0 + }, + "accuracy": 0.927102989373324, + "macro avg": { + "precision": 0.8993690266994916, + "recall": 0.8272187761746579, + "f1-score": 0.8575033232179177, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9243678106713654, + "recall": 0.927102989373324, + "f1-score": 0.9234929455838224, + "support": 20138.0 + }, + "cross-entropy": 0.17790377870690235, + "roc-auc": 0.9559529233208351, + "score": 0.9559529233208351 + }, + "test": { + "0": { + "precision": 0.9330988495640068, + "recall": 0.9685859891990568, + "f1-score": 0.9505113085019035, + "support": 13147.0 + }, + "1": { + "precision": 0.8440332326283988, + "recall": 0.7099745870393901, + "f1-score": 0.7712215320910973, + "support": 3148.0 + }, + "accuracy": 0.9186253451979135, + "macro avg": { + "precision": 0.8885660410962029, + "recall": 0.8392802881192234, + "f1-score": 0.8608664202965004, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9158924327420801, + "recall": 0.9186253451979135, + "f1-score": 0.9158746582324209, + "support": 16295.0 + }, + "cross-entropy": 0.18384793846446465, + "roc-auc": 0.9601378276664159, + "score": 0.9601378276664159 + } + }, + "time": "0:04:18.975261", + "diversity": { + "train": { + "mean_centered_corr": 0.054093651473522186, + "mean_pairwise_disagreement": 0.03080082888500299, + "member_std": 0.028895577415823936 + }, + "val": { + "mean_centered_corr": 0.0648474395275116, + "mean_pairwise_disagreement": 0.027818936435370138, + "member_std": 0.030719202011823654 + }, + "test": { + "mean_centered_corr": 0.07701285928487778, + "mean_pairwise_disagreement": 0.03625110116897128, + "member_std": 0.03344917297363281 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31 + ], + "head_selection_time": "0:00:00.950216" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.0001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.0001-evaluation/0.toml new file mode 100644 index 000000000..e37dbdda4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.0001-evaluation/0.toml @@ -0,0 +1,41 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.0001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.0001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.0001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.0001-evaluation/0/report.json new file mode 100644 index 000000000..3adca43df --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.0001-evaluation/0/report.json @@ -0,0 +1,271 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3057696, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9447221607027979, + "recall": 0.9845469882563705, + "f1-score": 0.9642235336223437, + "support": 181971.0 + }, + "1": { + "precision": 0.9189111252090663, + "recall": 0.7524616874070226, + "f1-score": 0.8273981850519948, + "support": 42349.0 + }, + "accuracy": 0.9407319900142653, + "macro avg": { + "precision": 0.931816642955932, + "recall": 0.8685043378316966, + "f1-score": 0.8958108593371692, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9398493382075944, + "recall": 0.9407319900142653, + "f1-score": 0.9383925034573752, + "support": 224320.0 + }, + "cross-entropy": 0.13832211550339593, + "roc-auc": 0.9790085111590701, + "score": 0.9790085111590701 + }, + "val": { + "0": { + "precision": 0.9330724292439453, + "recall": 0.9826772594229736, + "f1-score": 0.9572326312114512, + "support": 16741.0 + }, + "1": { + "precision": 0.8843238930993219, + "recall": 0.652634677656756, + "f1-score": 0.7510162601626016, + "support": 3397.0 + }, + "accuracy": 0.9270036746449498, + "macro avg": { + "precision": 0.9086981611716336, + "recall": 0.8176559685398648, + "f1-score": 0.8541244456870264, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9248492304514493, + "recall": 0.9270036746449498, + "f1-score": 0.92244680280481, + "support": 20138.0 + }, + "cross-entropy": 0.17414082376745435, + "roc-auc": 0.9582141658213201, + "score": 0.9582141658213201 + }, + "test": { + "0": { + "precision": 0.9287518948964123, + "recall": 0.978626302578535, + "f1-score": 0.953037037037037, + "support": 13147.0 + }, + "1": { + "precision": 0.8849303849303849, + "recall": 0.6864675984752223, + "f1-score": 0.7731663685152057, + "support": 3148.0 + }, + "accuracy": 0.9221847192390303, + "macro avg": { + "precision": 0.9068411399133987, + "recall": 0.8325469505268787, + "f1-score": 0.8631017027761214, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9202861008876332, + "recall": 0.9221847192390303, + "f1-score": 0.9182881653275111, + "support": 16295.0 + }, + "cross-entropy": 0.17803392077378757, + "roc-auc": 0.962170313614336, + "score": 0.962170313614336 + } + }, + "time": "0:04:18.975261", + "diversity": { + "train": { + "mean_centered_corr": 0.054093651473522186, + "mean_pairwise_disagreement": 0.03080082888500299, + "member_std": 0.028895577415823936 + }, + "val": { + "mean_centered_corr": 0.0648474395275116, + "mean_pairwise_disagreement": 0.027818936435370138, + "member_std": 0.030719202011823654 + }, + "test": { + "mean_centered_corr": 0.07701285928487778, + "mean_pairwise_disagreement": 0.03625110116897128, + "member_std": 0.03344917297363281 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.0001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.0001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.0001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.0001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..ece4c009a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.0001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,286 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3057696, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.944993059477376, + "recall": 0.9839205148073045, + "f1-score": 0.9640639891448909, + "support": 181971.0 + }, + "1": { + "precision": 0.9160473990761197, + "recall": 0.7539020992231221, + "f1-score": 0.8271029247946944, + "support": 42349.0 + }, + "accuracy": 0.9404957203994294, + "macro avg": { + "precision": 0.9305202292767478, + "recall": 0.8689113070152132, + "f1-score": 0.8955834569697927, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9395284563553503, + "recall": 0.9404957203994294, + "f1-score": 0.9382073374233928, + "support": 224320.0 + }, + "cross-entropy": 0.1396240616739436, + "roc-auc": 0.9782459886492416, + "score": 0.9782459886492416 + }, + "val": { + "0": { + "precision": 0.9338289540888712, + "recall": 0.9829161937757601, + "f1-score": 0.9577440195564868, + "support": 16741.0 + }, + "1": { + "precision": 0.88637266587207, + "recall": 0.6567559611421843, + "f1-score": 0.7544808927967535, + "support": 3397.0 + }, + "accuracy": 0.9278975072003178, + "macro avg": { + "precision": 0.9101008099804706, + "recall": 0.8198360774589721, + "f1-score": 0.8561124561766201, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9258237395158017, + "recall": 0.9278975072003178, + "f1-score": 0.9234563623113375, + "support": 20138.0 + }, + "cross-entropy": 0.1730646434975347, + "roc-auc": 0.9589703663198782, + "score": 0.9589703663198782 + }, + "test": { + "0": { + "precision": 0.9299920445505171, + "recall": 0.9780938617175021, + "f1-score": 0.9534366426929636, + "support": 13147.0 + }, + "1": { + "precision": 0.8833063209076175, + "recall": 0.6925031766200762, + "f1-score": 0.7763532763532763, + "support": 3148.0 + }, + "accuracy": 0.9229211414544338, + "macro avg": { + "precision": 0.9066491827290672, + "recall": 0.8352985191687892, + "f1-score": 0.8648949595231199, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9209729185592407, + "recall": 0.9229211414544338, + "f1-score": 0.9192262445808228, + "support": 16295.0 + }, + "cross-entropy": 0.17706493879213278, + "roc-auc": 0.9625895781732687, + "score": 0.9625895781732687 + } + }, + "time": "0:04:18.975261", + "diversity": { + "train": { + "mean_centered_corr": 0.054093651473522186, + "mean_pairwise_disagreement": 0.03080082888500299, + "member_std": 0.028895577415823936 + }, + "val": { + "mean_centered_corr": 0.0648474395275116, + "mean_pairwise_disagreement": 0.027818936435370138, + "member_std": 0.030719202011823654 + }, + "test": { + "mean_centered_corr": 0.07701285928487778, + "mean_pairwise_disagreement": 0.03625110116897128, + "member_std": 0.03344917297363281 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31, + 24, + 13, + 15, + 17, + 8, + 11, + 23, + 24, + 13, + 6, + 12 + ], + "head_selection_time": "0:00:12.138914" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..a409eec65 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.001-best-head-evaluation/0/report.json @@ -0,0 +1,275 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3057696, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9443811031934547, + "recall": 0.9831951244978595, + "f1-score": 0.9633973307917431, + "support": 181971.0 + }, + "1": { + "precision": 0.9123028391167193, + "recall": 0.7511865687501476, + "f1-score": 0.8239422939950012, + "support": 42349.0 + }, + "accuracy": 0.9393946148359487, + "macro avg": { + "precision": 0.928341971155087, + "recall": 0.8671908466240035, + "f1-score": 0.8936698123933722, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9383251010296455, + "recall": 0.9393946148359487, + "f1-score": 0.9370698461568233, + "support": 224320.0 + }, + "cross-entropy": 0.13960149767979568, + "roc-auc": 0.9778102042377117, + "score": 0.9778102042377117 + }, + "val": { + "0": { + "precision": 0.9351369902190699, + "recall": 0.9765844334269159, + "f1-score": 0.955411407199626, + "support": 16741.0 + }, + "1": { + "precision": 0.8523540489642184, + "recall": 0.6661760376803062, + "f1-score": 0.7478519497686715, + "support": 3397.0 + }, + "accuracy": 0.9242228622504718, + "macro avg": { + "precision": 0.8937455195916442, + "recall": 0.8213802355536111, + "f1-score": 0.8516316784841488, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9211726615149916, + "recall": 0.9242228622504718, + "f1-score": 0.9203990188346965, + "support": 20138.0 + }, + "cross-entropy": 0.17988717365824028, + "roc-auc": 0.9553321388843028, + "score": 0.9553321388843028 + }, + "test": { + "0": { + "precision": 0.9336227831334167, + "recall": 0.9650110291321214, + "f1-score": 0.9490574506283662, + "support": 13147.0 + }, + "1": { + "precision": 0.8300073909830007, + "recall": 0.7134688691232529, + "f1-score": 0.7673385719166382, + "support": 3148.0 + }, + "accuracy": 0.9164160785517029, + "macro avg": { + "precision": 0.8818150870582087, + "recall": 0.8392399491276872, + "f1-score": 0.8581980112725023, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9136055229622286, + "recall": 0.9164160785517029, + "f1-score": 0.9139515267140048, + "support": 16295.0 + }, + "cross-entropy": 0.1855192604147141, + "roc-auc": 0.9596838539362689, + "score": 0.9596838539362689 + } + }, + "time": "0:04:29.924516", + "diversity": { + "train": { + "mean_centered_corr": 0.05251581221818924, + "mean_pairwise_disagreement": 0.03326311814055957, + "member_std": 0.03071427531540394 + }, + "val": { + "mean_centered_corr": 0.06350768357515335, + "mean_pairwise_disagreement": 0.03187952722985593, + "member_std": 0.033710550516843796 + }, + "test": { + "mean_centered_corr": 0.07430540025234222, + "mean_pairwise_disagreement": 0.04065862277167942, + "member_std": 0.03548940643668175 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31 + ], + "head_selection_time": "0:00:00.832394" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.001-evaluation/0.toml new file mode 100644 index 000000000..e20b29645 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.001-evaluation/0.toml @@ -0,0 +1,41 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.001-evaluation/0/report.json new file mode 100644 index 000000000..8511fd467 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.001-evaluation/0/report.json @@ -0,0 +1,271 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3057696, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9489507453837032, + "recall": 0.9850635540827934, + "f1-score": 0.966669992207449, + "support": 181971.0 + }, + "1": { + "precision": 0.9232723577235772, + "recall": 0.7722968665139672, + "f1-score": 0.8410630938757666, + "support": 42349.0 + }, + "accuracy": 0.9448956847360913, + "macro avg": { + "precision": 0.9361115515536402, + "recall": 0.8786802102983804, + "f1-score": 0.9038665430416077, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9441029652525571, + "recall": 0.9448956847360913, + "f1-score": 0.9429568746189663, + "support": 224320.0 + }, + "cross-entropy": 0.13130891322714677, + "roc-auc": 0.9818073079002588, + "score": 0.9818073079002588 + }, + "val": { + "0": { + "precision": 0.9361495923835585, + "recall": 0.9808852517770742, + "f1-score": 0.95799544950703, + "support": 16741.0 + }, + "1": { + "precision": 0.8767809010396611, + "recall": 0.6702973211657345, + "f1-score": 0.7597597597597597, + "support": 3397.0 + }, + "accuracy": 0.9284933955705631, + "macro avg": { + "precision": 0.9064652467116099, + "recall": 0.8255912864714043, + "f1-score": 0.8588776046333948, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9261349213886623, + "recall": 0.9284933955705631, + "f1-score": 0.9245558508342979, + "support": 20138.0 + }, + "cross-entropy": 0.17334674258328744, + "roc-auc": 0.958447068787368, + "score": 0.958447068787368 + }, + "test": { + "0": { + "precision": 0.9326930073491959, + "recall": 0.974975279531452, + "f1-score": 0.9533655634064708, + "support": 13147.0 + }, + "1": { + "precision": 0.8710815047021944, + "recall": 0.7061626429479034, + "f1-score": 0.78, + "support": 3148.0 + }, + "accuracy": 0.9230438784903344, + "macro avg": { + "precision": 0.9018872560256952, + "recall": 0.8405689612396777, + "f1-score": 0.8666827817032354, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9207903985530768, + "recall": 0.9230438784903344, + "f1-score": 0.9198734005587526, + "support": 16295.0 + }, + "cross-entropy": 0.17722974814070125, + "roc-auc": 0.9625908587761747, + "score": 0.9625908587761747 + } + }, + "time": "0:04:29.924516", + "diversity": { + "train": { + "mean_centered_corr": 0.05251581221818924, + "mean_pairwise_disagreement": 0.03326311814055957, + "member_std": 0.03071427531540394 + }, + "val": { + "mean_centered_corr": 0.06350768357515335, + "mean_pairwise_disagreement": 0.03187952722985593, + "member_std": 0.033710550516843796 + }, + "test": { + "mean_centered_corr": 0.07430540025234222, + "mean_pairwise_disagreement": 0.04065862277167942, + "member_std": 0.03548940643668175 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..afbe38f9e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,283 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3057696, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9464479664251896, + "recall": 0.986470371652626, + "f1-score": 0.9660448233087664, + "support": 181971.0 + }, + "1": { + "precision": 0.9289548104114965, + "recall": 0.7601596259651939, + "f1-score": 0.8361232679246263, + "support": 42349.0 + }, + "accuracy": 0.943745542082739, + "macro avg": { + "precision": 0.937701388418343, + "recall": 0.87331499880891, + "f1-score": 0.9010840456166964, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9431454625734426, + "recall": 0.943745542082739, + "f1-score": 0.9415171487859286, + "support": 224320.0 + }, + "cross-entropy": 0.1327637198235955, + "roc-auc": 0.9809282833493577, + "score": 0.9809282833493577 + }, + "val": { + "0": { + "precision": 0.9345985567361782, + "recall": 0.9824980586583836, + "f1-score": 0.9579499126383226, + "support": 16741.0 + }, + "1": { + "precision": 0.8846002363135093, + "recall": 0.6611716220194289, + "f1-score": 0.7567385444743935, + "support": 3397.0 + }, + "accuracy": 0.9282947661138147, + "macro avg": { + "precision": 0.9095993965248437, + "recall": 0.8218348403389062, + "f1-score": 0.8573442285563581, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9261645367502904, + "recall": 0.9282947661138147, + "f1-score": 0.924008358479376, + "support": 20138.0 + }, + "cross-entropy": 0.17278712862311624, + "roc-auc": 0.9591161746546817, + "score": 0.9591161746546817 + }, + "test": { + "0": { + "precision": 0.9311094996374184, + "recall": 0.9766486650946984, + "f1-score": 0.9533355607528678, + "support": 13147.0 + }, + "1": { + "precision": 0.8774451097804391, + "recall": 0.6982210927573063, + "f1-score": 0.7776401910490005, + "support": 3148.0 + }, + "accuracy": 0.9228597729364836, + "macro avg": { + "precision": 0.9042773047089288, + "recall": 0.8374348789260023, + "f1-score": 0.8654878759009341, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9207421784180401, + "recall": 0.9228597729364836, + "f1-score": 0.9193933070659839, + "support": 16295.0 + }, + "cross-entropy": 0.1777732713928136, + "roc-auc": 0.9623163265079293, + "score": 0.9623163265079293 + } + }, + "time": "0:04:29.924516", + "diversity": { + "train": { + "mean_centered_corr": 0.05251581221818924, + "mean_pairwise_disagreement": 0.03326311814055957, + "member_std": 0.03071427531540394 + }, + "val": { + "mean_centered_corr": 0.06350768357515335, + "mean_pairwise_disagreement": 0.03187952722985593, + "member_std": 0.033710550516843796 + }, + "test": { + "mean_centered_corr": 0.07430540025234222, + "mean_pairwise_disagreement": 0.04065862277167942, + "member_std": 0.03548940643668175 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31, + 26, + 23, + 24, + 5, + 9, + 2, + 21, + 6 + ], + "head_selection_time": "0:00:10.170223" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise1e-05-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise1e-05-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise1e-05-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise1e-05-best-head-evaluation/0/report.json new file mode 100644 index 000000000..289c9b3b9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise1e-05-best-head-evaluation/0/report.json @@ -0,0 +1,275 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3057696, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9480838502035509, + "recall": 0.977765687939287, + "f1-score": 0.9626960358620167, + "support": 181971.0 + }, + "1": { + "precision": 0.8896103896103896, + "recall": 0.7699355356679024, + "f1-score": 0.8254579055961317, + "support": 42349.0 + }, + "accuracy": 0.9385297788873038, + "macro avg": { + "precision": 0.9188471199069703, + "recall": 0.8738506118035947, + "f1-score": 0.8940769707290742, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9370447427558878, + "recall": 0.9385297788873038, + "f1-score": 0.9367870773267547, + "support": 224320.0 + }, + "cross-entropy": 0.14147821261922355, + "roc-auc": 0.9765004386230669, + "score": 0.9765004386230669 + }, + "val": { + "0": { + "precision": 0.9363126573586633, + "recall": 0.977420703661669, + "f1-score": 0.9564251687757548, + "support": 16741.0 + }, + "1": { + "precision": 0.8580015026296018, + "recall": 0.6723579629084486, + "f1-score": 0.7539197887440172, + "support": 3397.0 + }, + "accuracy": 0.9259608699970205, + "macro avg": { + "precision": 0.8971570799941326, + "recall": 0.8248893332850589, + "f1-score": 0.855172478759886, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9231026567322546, + "recall": 0.9259608699970205, + "f1-score": 0.9222653328452844, + "support": 20138.0 + }, + "cross-entropy": 0.18154143168207998, + "roc-auc": 0.9558571420859494, + "score": 0.9558571420859494 + }, + "test": { + "0": { + "precision": 0.9311504166057594, + "recall": 0.9690423670799422, + "f1-score": 0.9497185880949719, + "support": 13147.0 + }, + "1": { + "precision": 0.8442403367776502, + "recall": 0.7007623888182973, + "f1-score": 0.7658392640166638, + "support": 3148.0 + }, + "accuracy": 0.9172138692850568, + "macro avg": { + "precision": 0.8876953766917048, + "recall": 0.8349023779491198, + "f1-score": 0.8577789260558178, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9143604238902707, + "recall": 0.9172138692850568, + "f1-score": 0.9141952918569531, + "support": 16295.0 + }, + "cross-entropy": 0.1895139723221945, + "roc-auc": 0.958774227194806, + "score": 0.958774227194806 + } + }, + "time": "0:04:19.140003", + "diversity": { + "train": { + "mean_centered_corr": 0.05099165812134743, + "mean_pairwise_disagreement": 0.03067784069992177, + "member_std": 0.028426973149180412 + }, + "val": { + "mean_centered_corr": 0.059033043682575226, + "mean_pairwise_disagreement": 0.02795969904433602, + "member_std": 0.03006986528635025 + }, + "test": { + "mean_centered_corr": 0.06953592598438263, + "mean_pairwise_disagreement": 0.03653245602747726, + "member_std": 0.032674212008714676 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 7 + ], + "head_selection_time": "0:00:00.881435" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise1e-05-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise1e-05-evaluation/0.toml new file mode 100644 index 000000000..72e7526d1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise1e-05-evaluation/0.toml @@ -0,0 +1,41 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise1e-05-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise1e-05-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise1e-05-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise1e-05-evaluation/0/report.json new file mode 100644 index 000000000..9cd91ee41 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise1e-05-evaluation/0/report.json @@ -0,0 +1,271 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3057696, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9455209967307316, + "recall": 0.9838051118035291, + "f1-score": 0.9642832134873826, + "support": 181971.0 + }, + "1": { + "precision": 0.9157542666018695, + "recall": 0.7564287232284115, + "f1-score": 0.8285012285012285, + "support": 42349.0 + }, + "accuracy": 0.9408791012838802, + "macro avg": { + "precision": 0.9306376316663005, + "recall": 0.8701169175159703, + "f1-score": 0.8963922209943056, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9399013852193765, + "recall": 0.9408791012838802, + "f1-score": 0.9386491581994965, + "support": 224320.0 + }, + "cross-entropy": 0.13842434069038026, + "roc-auc": 0.9789222515567924, + "score": 0.9789222515567924 + }, + "val": { + "0": { + "precision": 0.9340303426331041, + "recall": 0.9819007227764172, + "f1-score": 0.957367501456028, + "support": 16741.0 + }, + "1": { + "precision": 0.8806616778259158, + "recall": 0.6582278481012658, + "f1-score": 0.7533692722371967, + "support": 3397.0 + }, + "accuracy": 0.9273016188300724, + "macro avg": { + "precision": 0.9073460102295099, + "recall": 0.8200642854388415, + "f1-score": 0.8553683868466124, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9250277925114427, + "recall": 0.9273016188300724, + "f1-score": 0.9229558426688411, + "support": 20138.0 + }, + "cross-entropy": 0.17369357261885118, + "roc-auc": 0.9583854185194205, + "score": 0.9583854185194205 + }, + "test": { + "0": { + "precision": 0.9296467863346844, + "recall": 0.9769529170152886, + "f1-score": 0.9527129770426139, + "support": 13147.0 + }, + "1": { + "precision": 0.8777732956837434, + "recall": 0.6912325285895807, + "f1-score": 0.7734138972809668, + "support": 3148.0 + }, + "accuracy": 0.9217551396133783, + "macro avg": { + "precision": 0.9037100410092139, + "recall": 0.8340927228024346, + "f1-score": 0.8630634371617903, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9196254455203756, + "recall": 0.9217551396133783, + "f1-score": 0.9180745294765099, + "support": 16295.0 + }, + "cross-entropy": 0.1775633543432209, + "roc-auc": 0.9623873274822505, + "score": 0.9623873274822505 + } + }, + "time": "0:04:19.140003", + "diversity": { + "train": { + "mean_centered_corr": 0.05099165812134743, + "mean_pairwise_disagreement": 0.03067784069992177, + "member_std": 0.028426973149180412 + }, + "val": { + "mean_centered_corr": 0.059033043682575226, + "mean_pairwise_disagreement": 0.02795969904433602, + "member_std": 0.03006986528635025 + }, + "test": { + "mean_centered_corr": 0.06953592598438263, + "mean_pairwise_disagreement": 0.03653245602747726, + "member_std": 0.032674212008714676 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise1e-05-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise1e-05-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise1e-05-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise1e-05-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..9549e7e89 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise1e-05-greedy-heads-evaluation/0/report.json @@ -0,0 +1,292 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3057696, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9446692233434791, + "recall": 0.98411285314693, + "f1-score": 0.9639877267588954, + "support": 181971.0 + }, + "1": { + "precision": 0.9168081494057725, + "recall": 0.7523200075562587, + "f1-score": 0.8264591439688715, + "support": 42349.0 + }, + "accuracy": 0.9403530670470756, + "macro avg": { + "precision": 0.9307386863746259, + "recall": 0.8682164303515943, + "f1-score": 0.8952234353638835, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9394093774974203, + "recall": 0.9403530670470756, + "f1-score": 0.9380239341743076, + "support": 224320.0 + }, + "cross-entropy": 0.13997136532599735, + "roc-auc": 0.978227078576783, + "score": 0.978227078576783 + }, + "val": { + "0": { + "precision": 0.9330876098667423, + "recall": 0.9829161937757601, + "f1-score": 0.9573539678845706, + "support": 16741.0 + }, + "1": { + "precision": 0.8857371154614463, + "recall": 0.652634677656756, + "f1-score": 0.7515254237288136, + "support": 3397.0 + }, + "accuracy": 0.9272023041016982, + "macro avg": { + "precision": 0.9094123626640943, + "recall": 0.8177754357162581, + "f1-score": 0.8544396958066921, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9251002412355581, + "recall": 0.9272023041016982, + "f1-score": 0.9226335604708698, + "support": 20138.0 + }, + "cross-entropy": 0.17266928890233252, + "roc-auc": 0.9591292133522524, + "score": 0.9591292133522524 + }, + "test": { + "0": { + "precision": 0.9302510672165545, + "recall": 0.977941735757207, + "f1-score": 0.9535004449718184, + "support": 13147.0 + }, + "1": { + "precision": 0.8827809215844786, + "recall": 0.6937738246505718, + "f1-score": 0.7769477054429029, + "support": 3148.0 + }, + "accuracy": 0.9230438784903344, + "macro avg": { + "precision": 0.9065159944005166, + "recall": 0.8358577802038893, + "f1-score": 0.8652240752073607, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9210804002358993, + "recall": 0.9230438784903344, + "f1-score": 0.9193925576421451, + "support": 16295.0 + }, + "cross-entropy": 0.17693632251110164, + "roc-auc": 0.9626703044809795, + "score": 0.9626703044809795 + } + }, + "time": "0:04:19.140003", + "diversity": { + "train": { + "mean_centered_corr": 0.05099165812134743, + "mean_pairwise_disagreement": 0.03067784069992177, + "member_std": 0.028426973149180412 + }, + "val": { + "mean_centered_corr": 0.059033043682575226, + "mean_pairwise_disagreement": 0.02795969904433602, + "member_std": 0.03006986528635025 + }, + "test": { + "mean_centered_corr": 0.06953592598438263, + "mean_pairwise_disagreement": 0.03653245602747726, + "member_std": 0.032674212008714676 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 7, + 31, + 15, + 17, + 24, + 0, + 6, + 19, + 21, + 29, + 5, + 17, + 12, + 24, + 31, + 13, + 15, + 0 + ], + "head_selection_time": "0:00:16.480622" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.0001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.0001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.0001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.0001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..2c5ded4f4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.0001-best-head-evaluation/0/report.json @@ -0,0 +1,275 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9470929244882358, + "recall": 0.9803869847393266, + "f1-score": 0.9634524044186542, + "support": 181971.0 + }, + "1": { + "precision": 0.9007287494437027, + "recall": 0.7646697678811778, + "f1-score": 0.8271414158184442, + "support": 42349.0 + }, + "accuracy": 0.9396620898716119, + "macro avg": { + "precision": 0.9239108369659692, + "recall": 0.8725283763102523, + "f1-score": 0.8952969101185493, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9383399089347366, + "recall": 0.9396620898716119, + "f1-score": 0.9377184794176276, + "support": 224320.0 + }, + "cross-entropy": 0.1374541129990154, + "roc-auc": 0.9783636099059336, + "score": 0.9783636099059336 + }, + "val": { + "0": { + "precision": 0.9391985260248733, + "recall": 0.9743742906636401, + "f1-score": 0.9564631035797003, + "support": 16741.0 + }, + "1": { + "precision": 0.8451263537906137, + "recall": 0.6891374742419782, + "f1-score": 0.7592022052862007, + "support": 3397.0 + }, + "accuracy": 0.9262588141821432, + "macro avg": { + "precision": 0.8921624399077435, + "recall": 0.8317558824528092, + "f1-score": 0.8578326544329505, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9233298613570919, + "recall": 0.9262588141821432, + "f1-score": 0.9231879386426154, + "support": 20138.0 + }, + "cross-entropy": 0.18034048370527975, + "roc-auc": 0.9557610707114682, + "score": 0.9557610707114682 + }, + "test": { + "0": { + "precision": 0.9347106830380119, + "recall": 0.9669886666159581, + "f1-score": 0.9505757439808584, + "support": 13147.0 + }, + "1": { + "precision": 0.8389012620638456, + "recall": 0.7179161372299873, + "f1-score": 0.7737076343717905, + "support": 3148.0 + }, + "accuracy": 0.9188708192697146, + "macro avg": { + "precision": 0.8868059725509287, + "recall": 0.8424524019229727, + "f1-score": 0.8621416891763245, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.916201443564144, + "recall": 0.9188708192697146, + "f1-score": 0.9164069309063357, + "support": 16295.0 + }, + "cross-entropy": 0.18574135097736744, + "roc-auc": 0.9604946447119459, + "score": 0.9604946447119459 + } + }, + "time": "0:07:11.833138", + "diversity": { + "train": { + "mean_centered_corr": 0.05171258747577667, + "mean_pairwise_disagreement": 0.03286417948437716, + "member_std": 0.03040449135005474 + }, + "val": { + "mean_centered_corr": 0.061806585639715195, + "mean_pairwise_disagreement": 0.03182976975001522, + "member_std": 0.0331539586186409 + }, + "test": { + "mean_centered_corr": 0.0732795000076294, + "mean_pairwise_disagreement": 0.03959877856852983, + "member_std": 0.03512473404407501 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31 + ], + "head_selection_time": "0:00:00.946516" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.0001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.0001-evaluation/0.toml new file mode 100644 index 000000000..d6037d45d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.0001-evaluation/0.toml @@ -0,0 +1,41 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.0001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.0001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.0001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.0001-evaluation/0/report.json new file mode 100644 index 000000000..e0392c3f8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.0001-evaluation/0/report.json @@ -0,0 +1,271 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9493414440624174, + "recall": 0.9862725379318683, + "f1-score": 0.9674546723770353, + "support": 181971.0 + }, + "1": { + "precision": 0.9291749362064077, + "recall": 0.7738553448723701, + "f1-score": 0.8444324198971901, + "support": 42349.0 + }, + "accuracy": 0.9461706490727532, + "macro avg": { + "precision": 0.9392581901344126, + "recall": 0.8800639414021192, + "f1-score": 0.9059435461371127, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9455342425592338, + "recall": 0.9461706490727532, + "f1-score": 0.9442295057834682, + "support": 224320.0 + }, + "cross-entropy": 0.1285332583359511, + "roc-auc": 0.9826472071386247, + "score": 0.9826472071386247 + }, + "val": { + "0": { + "precision": 0.9352083807788658, + "recall": 0.9811839197180575, + "f1-score": 0.9576446582130884, + "support": 16741.0 + }, + "1": { + "precision": 0.8776223776223776, + "recall": 0.6649985281130409, + "f1-score": 0.7566571763523698, + "support": 3397.0 + }, + "accuracy": 0.9278478498361307, + "macro avg": { + "precision": 0.9064153792006218, + "recall": 0.8230912239155492, + "f1-score": 0.8571509172827292, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9254944244414645, + "recall": 0.9278478498361307, + "f1-score": 0.9237408705538938, + "support": 20138.0 + }, + "cross-entropy": 0.173981665844509, + "roc-auc": 0.9583231528038466, + "score": 0.9583231528038466 + }, + "test": { + "0": { + "precision": 0.9317207815791385, + "recall": 0.9756598463527801, + "f1-score": 0.9531842163929554, + "support": 13147.0 + }, + "1": { + "precision": 0.8734177215189873, + "recall": 0.7013977128335451, + "f1-score": 0.7780126849894292, + "support": 3148.0 + }, + "accuracy": 0.9226756673826327, + "macro avg": { + "precision": 0.9025692515490629, + "recall": 0.8385287795931626, + "f1-score": 0.8655984506911922, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9204573245021606, + "recall": 0.9226756673826327, + "f1-score": 0.9193431620291443, + "support": 16295.0 + }, + "cross-entropy": 0.17786928687791626, + "roc-auc": 0.96242593886798, + "score": 0.96242593886798 + } + }, + "time": "0:07:11.833138", + "diversity": { + "train": { + "mean_centered_corr": 0.05171258747577667, + "mean_pairwise_disagreement": 0.03286417948437716, + "member_std": 0.03040449135005474 + }, + "val": { + "mean_centered_corr": 0.061806585639715195, + "mean_pairwise_disagreement": 0.03182976975001522, + "member_std": 0.0331539586186409 + }, + "test": { + "mean_centered_corr": 0.0732795000076294, + "mean_pairwise_disagreement": 0.03959877856852983, + "member_std": 0.03512473404407501 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.0001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.0001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.0001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.0001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..460c6ffd1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.0001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,285 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.946255084192114, + "recall": 0.9869869374790489, + "f1-score": 0.9661919171541927, + "support": 181971.0 + }, + "1": { + "precision": 0.931394136052845, + "recall": 0.7591206403929255, + "f1-score": 0.836479542054251, + "support": 42349.0 + }, + "accuracy": 0.9439684379457918, + "macro avg": { + "precision": 0.9388246101224795, + "recall": 0.8730537889359872, + "f1-score": 0.9013357296042219, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9434495104904829, + "recall": 0.9439684379457918, + "f1-score": 0.9417037334295696, + "support": 224320.0 + }, + "cross-entropy": 0.1317045399668242, + "roc-auc": 0.9813354497094697, + "score": 0.9813354497094697 + }, + "val": { + "0": { + "precision": 0.9333144315282109, + "recall": 0.9831551281285467, + "f1-score": 0.9575866883872469, + "support": 16741.0 + }, + "1": { + "precision": 0.8873351977626848, + "recall": 0.6538121872240212, + "f1-score": 0.7528813559322034, + "support": 3397.0 + }, + "accuracy": 0.9275995630151952, + "macro avg": { + "precision": 0.9103248146454479, + "recall": 0.818483657676284, + "f1-score": 0.8552340221597252, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.925558375460007, + "recall": 0.9275995630151952, + "f1-score": 0.9230557511367858, + "support": 20138.0 + }, + "cross-entropy": 0.1731441036701711, + "roc-auc": 0.959029757015826, + "score": 0.959029757015826 + }, + "test": { + "0": { + "precision": 0.9292688218702538, + "recall": 0.9773332319160265, + "f1-score": 0.9526951879587752, + "support": 13147.0 + }, + "1": { + "precision": 0.8792544570502431, + "recall": 0.6893265565438373, + "f1-score": 0.7727920227920227, + "support": 3148.0 + }, + "accuracy": 0.921693771095428, + "macro avg": { + "precision": 0.9042616394602485, + "recall": 0.8333298942299319, + "f1-score": 0.862743605375399, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9196066420326721, + "recall": 0.921693771095428, + "f1-score": 0.9179400382843391, + "support": 16295.0 + }, + "cross-entropy": 0.17812450933232857, + "roc-auc": 0.9624579176971494, + "score": 0.9624579176971494 + } + }, + "time": "0:07:11.833138", + "diversity": { + "train": { + "mean_centered_corr": 0.05171258747577667, + "mean_pairwise_disagreement": 0.03286417948437716, + "member_std": 0.03040449135005474 + }, + "val": { + "mean_centered_corr": 0.061806585639715195, + "mean_pairwise_disagreement": 0.03182976975001522, + "member_std": 0.0331539586186409 + }, + "test": { + "mean_centered_corr": 0.0732795000076294, + "mean_pairwise_disagreement": 0.03959877856852983, + "member_std": 0.03512473404407501 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31, + 22, + 6, + 20, + 26, + 21, + 24, + 7, + 22, + 13, + 5 + ], + "head_selection_time": "0:00:11.484544" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..e1fb60dd6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.001-best-head-evaluation/0/report.json @@ -0,0 +1,275 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1100, + "metrics": { + "train": { + "0": { + "precision": 0.9382483830460208, + "recall": 0.9813321902940578, + "f1-score": 0.9593067918710281, + "support": 181971.0 + }, + "1": { + "precision": 0.9000676609890271, + "recall": 0.7224727856619991, + "f1-score": 0.8015509156165675, + "support": 42349.0 + }, + "accuracy": 0.9324625534950072, + "macro avg": { + "precision": 0.919158022017524, + "recall": 0.8519024879780285, + "f1-score": 0.8804288537437979, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9310403079818641, + "recall": 0.9324625534950072, + "f1-score": 0.9295243221692621, + "support": 224320.0 + }, + "cross-entropy": 0.15611302140645272, + "roc-auc": 0.9712153076924243, + "score": 0.9712153076924243 + }, + "val": { + "0": { + "precision": 0.9314890246662141, + "recall": 0.9835135296577265, + "f1-score": 0.956794607316152, + "support": 16741.0 + }, + "1": { + "precision": 0.8878960194963444, + "recall": 0.6435089785104504, + "f1-score": 0.7462024236217785, + "support": 3397.0 + }, + "accuracy": 0.926159499453769, + "macro avg": { + "precision": 0.9096925220812793, + "recall": 0.8135112540840885, + "f1-score": 0.8514985154689652, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9241354921127308, + "recall": 0.926159499453769, + "f1-score": 0.9212706402881558, + "support": 20138.0 + }, + "cross-entropy": 0.17654285581424872, + "roc-auc": 0.9576809947504603, + "score": 0.9576809947504603 + }, + "test": { + "0": { + "precision": 0.9285455991899906, + "recall": 0.9765726021145509, + "f1-score": 0.9519537332245867, + "support": 13147.0 + }, + "1": { + "precision": 0.8752025931928687, + "recall": 0.6861499364675985, + "f1-score": 0.7692307692307693, + "support": 3148.0 + }, + "accuracy": 0.9204664007364222, + "macro avg": { + "precision": 0.9018740961914297, + "recall": 0.8313612692910747, + "f1-score": 0.860592251227678, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9182403655061034, + "recall": 0.9204664007364222, + "f1-score": 0.9166538319878553, + "support": 16295.0 + }, + "cross-entropy": 0.18087018625189427, + "roc-auc": 0.9611084763444616, + "score": 0.9611084763444616 + } + }, + "time": "0:06:04.606143", + "diversity": { + "train": { + "mean_centered_corr": 0.02503632754087448, + "mean_pairwise_disagreement": 0.025175350737425797, + "member_std": 0.02292787656188011 + }, + "val": { + "mean_centered_corr": 0.025311479344964027, + "mean_pairwise_disagreement": 0.021320028897382257, + "member_std": 0.0236747395247221 + }, + "test": { + "mean_centered_corr": 0.034202832728624344, + "mean_pairwise_disagreement": 0.02758341664274614, + "member_std": 0.0252092145383358 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 5 + ], + "head_selection_time": "0:00:01.295715" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.001-evaluation/0.toml new file mode 100644 index 000000000..37f311a6f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.001-evaluation/0.toml @@ -0,0 +1,41 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.001-evaluation/0/report.json new file mode 100644 index 000000000..024a124ef --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.001-evaluation/0/report.json @@ -0,0 +1,271 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1100, + "metrics": { + "train": { + "0": { + "precision": 0.9370407971029661, + "recall": 0.984008440905419, + "f1-score": 0.9599504641307233, + "support": 181971.0 + }, + "1": { + "precision": 0.91242325749368, + "recall": 0.7159082859099388, + "f1-score": 0.8023075803485187, + "support": 42349.0 + }, + "accuracy": 0.9333942582025677, + "macro avg": { + "precision": 0.924732027298323, + "recall": 0.8499583634076788, + "f1-score": 0.881129022239621, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9323932927123024, + "recall": 0.9333942582025677, + "f1-score": 0.930189326090011, + "support": 224320.0 + }, + "cross-entropy": 0.15323431377042523, + "roc-auc": 0.9727788043541361, + "score": 0.9727788043541361 + }, + "val": { + "0": { + "precision": 0.9285593744726331, + "recall": 0.9860223403619855, + "f1-score": 0.9564285300422968, + "support": 16741.0 + }, + "1": { + "precision": 0.9008894536213469, + "recall": 0.6261407123932882, + "f1-score": 0.7387981938172977, + "support": 3397.0 + }, + "accuracy": 0.9253153242625881, + "macro avg": { + "precision": 0.91472441404699, + "recall": 0.8060815263776369, + "f1-score": 0.8476133619297972, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9238918443737247, + "recall": 0.9253153242625881, + "f1-score": 0.9197173247509908, + "support": 20138.0 + }, + "cross-entropy": 0.1750501462455179, + "roc-auc": 0.9581143050478821, + "score": 0.9581143050478821 + }, + "test": { + "0": { + "precision": 0.9257745266781411, + "recall": 0.9818970107248802, + "f1-score": 0.9530102247979033, + "support": 13147.0 + }, + "1": { + "precision": 0.8987664823479371, + "recall": 0.6712198221092758, + "f1-score": 0.7685033642480451, + "support": 3148.0 + }, + "accuracy": 0.9218778766492789, + "macro avg": { + "precision": 0.912270504513039, + "recall": 0.8265584164170781, + "f1-score": 0.8607567945229742, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9205568940576144, + "recall": 0.9218778766492789, + "f1-score": 0.917365695984712, + "support": 16295.0 + }, + "cross-entropy": 0.1786217717894183, + "roc-auc": 0.9619551240015043, + "score": 0.9619551240015043 + } + }, + "time": "0:06:04.606143", + "diversity": { + "train": { + "mean_centered_corr": 0.02503632754087448, + "mean_pairwise_disagreement": 0.025175350737425797, + "member_std": 0.02292787656188011 + }, + "val": { + "mean_centered_corr": 0.025311479344964027, + "mean_pairwise_disagreement": 0.021320028897382257, + "member_std": 0.0236747395247221 + }, + "test": { + "mean_centered_corr": 0.034202832728624344, + "mean_pairwise_disagreement": 0.02758341664274614, + "member_std": 0.0252092145383358 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..df299eeba --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,278 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1100, + "metrics": { + "train": { + "0": { + "precision": 0.9377624907587523, + "recall": 0.9828544108676657, + "f1-score": 0.9597791205606775, + "support": 181971.0 + }, + "1": { + "precision": 0.9071400934551623, + "recall": 0.7197100285721032, + "f1-score": 0.8026281139727182, + "support": 42349.0 + }, + "accuracy": 0.933175820256776, + "macro avg": { + "precision": 0.9224512921069573, + "recall": 0.8512822197198844, + "f1-score": 0.8812036172666978, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9319813392635236, + "recall": 0.933175820256776, + "f1-score": 0.9301108431980104, + "support": 224320.0 + }, + "cross-entropy": 0.154500471418488, + "roc-auc": 0.9722444215234017, + "score": 0.9722444215234017 + }, + "val": { + "0": { + "precision": 0.9297020221934321, + "recall": 0.9859028731855922, + "f1-score": 0.9569780251637966, + "support": 16741.0 + }, + "1": { + "precision": 0.9010482180293501, + "recall": 0.632617015013247, + "f1-score": 0.7433414043583535, + "support": 3397.0 + }, + "accuracy": 0.9263084715463303, + "macro avg": { + "precision": 0.9153751201113911, + "recall": 0.8092599440994196, + "f1-score": 0.8501597147610751, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9248685246889436, + "recall": 0.9263084715463303, + "f1-score": 0.9209405040159125, + "support": 20138.0 + }, + "cross-entropy": 0.17378638383137032, + "roc-auc": 0.9589014449778305, + "score": 0.9589014449778305 + }, + "test": { + "0": { + "precision": 0.9251968503937008, + "recall": 0.9831140184072412, + "f1-score": 0.9532765423903824, + "support": 13147.0 + }, + "1": { + "precision": 0.9045161290322581, + "recall": 0.6680432020330368, + "f1-score": 0.7684999086424265, + "support": 3148.0 + }, + "accuracy": 0.9222460877569807, + "macro avg": { + "precision": 0.9148564897129794, + "recall": 0.825578610220139, + "f1-score": 0.8608882255164044, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9212015812408428, + "recall": 0.9222460877569807, + "f1-score": 0.9175798966071014, + "support": 16295.0 + }, + "cross-entropy": 0.17739934033244287, + "roc-auc": 0.9626523760402965, + "score": 0.9626523760402965 + } + }, + "time": "0:06:04.606143", + "diversity": { + "train": { + "mean_centered_corr": 0.02503632754087448, + "mean_pairwise_disagreement": 0.025175350737425797, + "member_std": 0.02292787656188011 + }, + "val": { + "mean_centered_corr": 0.025311479344964027, + "mean_pairwise_disagreement": 0.021320028897382257, + "member_std": 0.0236747395247221 + }, + "test": { + "mean_centered_corr": 0.034202832728624344, + "mean_pairwise_disagreement": 0.02758341664274614, + "member_std": 0.0252092145383358 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 5, + 31, + 3, + 21 + ], + "head_selection_time": "0:00:07.034698" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise1e-05-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise1e-05-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise1e-05-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise1e-05-best-head-evaluation/0/report.json new file mode 100644 index 000000000..7721a02fc --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise1e-05-best-head-evaluation/0/report.json @@ -0,0 +1,275 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9413658167129513, + "recall": 0.9835633150337142, + "f1-score": 0.9620020478421065, + "support": 181971.0 + }, + "1": { + "precision": 0.9125233972859148, + "recall": 0.7367588372806914, + "f1-score": 0.8152754732757607, + "support": 42349.0 + }, + "accuracy": 0.9369695078459344, + "macro avg": { + "precision": 0.926944606999433, + "recall": 0.8601610761572027, + "f1-score": 0.8886387605589336, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9359207042828712, + "recall": 0.9369695078459344, + "f1-score": 0.9343017816763158, + "support": 224320.0 + }, + "cross-entropy": 0.14393086493300278, + "roc-auc": 0.9759690841627111, + "score": 0.9759690841627111 + }, + "val": { + "0": { + "precision": 0.9325333937061354, + "recall": 0.9841705991278896, + "f1-score": 0.9576564270975617, + "support": 16741.0 + }, + "1": { + "precision": 0.8927125506072875, + "recall": 0.6491021489549602, + "f1-score": 0.7516618373956026, + "support": 3397.0 + }, + "accuracy": 0.9276492203793822, + "macro avg": { + "precision": 0.9126229721567114, + "recall": 0.8166363740414249, + "f1-score": 0.8546591322465822, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.925816172333269, + "recall": 0.9276492203793822, + "f1-score": 0.9229080101138715, + "support": 20138.0 + }, + "cross-entropy": 0.17925081582806562, + "roc-auc": 0.9560453459701026, + "score": 0.9560453459701026 + }, + "test": { + "0": { + "precision": 0.9250304899921085, + "recall": 0.9807560660226667, + "f1-score": 0.9520785645721037, + "support": 13147.0 + }, + "1": { + "precision": 0.8926146010186757, + "recall": 0.6680432020330368, + "f1-score": 0.764171511627907, + "support": 3148.0 + }, + "accuracy": 0.9203436637005217, + "macro avg": { + "precision": 0.9088225455053921, + "recall": 0.8243996340278518, + "f1-score": 0.8581250381000054, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9187681261695637, + "recall": 0.9203436637005217, + "f1-score": 0.9157771590692911, + "support": 16295.0 + }, + "cross-entropy": 0.18465226452577868, + "roc-auc": 0.9600924508313724, + "score": 0.9600924508313724 + } + }, + "time": "0:06:58.103811", + "diversity": { + "train": { + "mean_centered_corr": 0.05266324058175087, + "mean_pairwise_disagreement": 0.03131294111810317, + "member_std": 0.028657343238592148 + }, + "val": { + "mean_centered_corr": 0.06352649629116058, + "mean_pairwise_disagreement": 0.02856399712948398, + "member_std": 0.030144035816192627 + }, + "test": { + "mean_centered_corr": 0.0760873481631279, + "mean_pairwise_disagreement": 0.03589971691296558, + "member_std": 0.032248713076114655 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31 + ], + "head_selection_time": "0:00:00.778477" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise1e-05-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise1e-05-evaluation/0.toml new file mode 100644 index 000000000..601f75de8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise1e-05-evaluation/0.toml @@ -0,0 +1,41 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise1e-05-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise1e-05-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise1e-05-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise1e-05-evaluation/0/report.json new file mode 100644 index 000000000..4bb9206ba --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise1e-05-evaluation/0/report.json @@ -0,0 +1,271 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9463105683240804, + "recall": 0.9844810436827847, + "f1-score": 0.9650185034555944, + "support": 181971.0 + }, + "1": { + "precision": 0.9193350281356223, + "recall": 0.7599943328059694, + "f1-score": 0.8321052767651698, + "support": 42349.0 + }, + "accuracy": 0.9421005706134095, + "macro avg": { + "precision": 0.9328227982298514, + "recall": 0.872237688244377, + "f1-score": 0.898561890110382, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9412179009228634, + "recall": 0.9421005706134095, + "f1-score": 0.9399260362787364, + "support": 224320.0 + }, + "cross-entropy": 0.1363573248497679, + "roc-auc": 0.9795959593437451, + "score": 0.9795959593437451 + }, + "val": { + "0": { + "precision": 0.9344513928368391, + "recall": 0.9818409891882205, + "f1-score": 0.9575602225393959, + "support": 16741.0 + }, + "1": { + "precision": 0.880690737833595, + "recall": 0.6605828672357963, + "f1-score": 0.7549201009251472, + "support": 3397.0 + }, + "accuracy": 0.9276492203793822, + "macro avg": { + "precision": 0.907571065335217, + "recall": 0.8212119282120084, + "f1-score": 0.8562401617322715, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9253827194310382, + "recall": 0.9276492203793822, + "f1-score": 0.9233776575814258, + "support": 20138.0 + }, + "cross-entropy": 0.17415890290359384, + "roc-auc": 0.9581626616470993, + "score": 0.9581626616470993 + }, + "test": { + "0": { + "precision": 0.9295326291419477, + "recall": 0.9772571689358789, + "f1-score": 0.9527976565686529, + "support": 13147.0 + }, + "1": { + "precision": 0.879094217549535, + "recall": 0.6905972045743329, + "f1-score": 0.7735278420209927, + "support": 3148.0 + }, + "accuracy": 0.9218778766492789, + "macro avg": { + "precision": 0.9043134233457413, + "recall": 0.8339271867551059, + "f1-score": 0.8631627492948228, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9197885285164236, + "recall": 0.9218778766492789, + "f1-score": 0.9181648626934744, + "support": 16295.0 + }, + "cross-entropy": 0.17842812985458048, + "roc-auc": 0.9620964494052155, + "score": 0.9620964494052155 + } + }, + "time": "0:06:58.103811", + "diversity": { + "train": { + "mean_centered_corr": 0.05266324058175087, + "mean_pairwise_disagreement": 0.03131294111810317, + "member_std": 0.028657343238592148 + }, + "val": { + "mean_centered_corr": 0.06352649629116058, + "mean_pairwise_disagreement": 0.02856399712948398, + "member_std": 0.030144035816192627 + }, + "test": { + "mean_centered_corr": 0.0760873481631279, + "mean_pairwise_disagreement": 0.03589971691296558, + "member_std": 0.032248713076114655 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise1e-05-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise1e-05-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise1e-05-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise1e-05-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..b14bbaa40 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise1e-05-greedy-heads-evaluation/0/report.json @@ -0,0 +1,284 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9441162844567987, + "recall": 0.985503184573366, + "f1-score": 0.9643658968753949, + "support": 181971.0 + }, + "1": { + "precision": 0.923251483765856, + "recall": 0.749344730690217, + "f1-score": 0.8272572046766856, + "support": 42349.0 + }, + "accuracy": 0.9409192225392297, + "macro avg": { + "precision": 0.9336838841113273, + "recall": 0.8674239576317915, + "f1-score": 0.8958115507760402, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9401772534098091, + "recall": 0.9409192225392297, + "f1-score": 0.9384813747377204, + "support": 224320.0 + }, + "cross-entropy": 0.13816220002148916, + "roc-auc": 0.9787552331834622, + "score": 0.9787552331834622 + }, + "val": { + "0": { + "precision": 0.931622965641953, + "recall": 0.984767935009856, + "f1-score": 0.9574585474925226, + "support": 16741.0 + }, + "1": { + "precision": 0.8955773955773956, + "recall": 0.6438033559022667, + "f1-score": 0.7491008734372324, + "support": 3397.0 + }, + "accuracy": 0.9272519614658854, + "macro avg": { + "precision": 0.9136001806096743, + "recall": 0.8142856454560614, + "f1-score": 0.8532797104648775, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.925542580225859, + "recall": 0.9272519614658854, + "f1-score": 0.9223115111053034, + "support": 20138.0 + }, + "cross-entropy": 0.17350924991795408, + "roc-auc": 0.9591329763748824, + "score": 0.9591329763748824 + }, + "test": { + "0": { + "precision": 0.9269169903610991, + "recall": 0.9801475621814862, + "f1-score": 0.9527893822322452, + "support": 13147.0 + }, + "1": { + "precision": 0.8909318846636022, + "recall": 0.6772554002541296, + "f1-score": 0.769536184804187, + "support": 3148.0 + }, + "accuracy": 0.9216324025774778, + "macro avg": { + "precision": 0.9089244375123506, + "recall": 0.828701481217808, + "f1-score": 0.8611627835182161, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9199650963607481, + "recall": 0.9216324025774778, + "f1-score": 0.9173870462087087, + "support": 16295.0 + }, + "cross-entropy": 0.17845179542990083, + "roc-auc": 0.962370812537228, + "score": 0.962370812537228 + } + }, + "time": "0:06:58.103811", + "diversity": { + "train": { + "mean_centered_corr": 0.05266324058175087, + "mean_pairwise_disagreement": 0.03131294111810317, + "member_std": 0.028657343238592148 + }, + "val": { + "mean_centered_corr": 0.06352649629116058, + "mean_pairwise_disagreement": 0.02856399712948398, + "member_std": 0.030144035816192627 + }, + "test": { + "mean_centered_corr": 0.0760873481631279, + "mean_pairwise_disagreement": 0.03589971691296558, + "member_std": 0.032248713076114655 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31, + 24, + 30, + 13, + 17, + 21, + 28, + 24, + 6, + 8 + ], + "head_selection_time": "0:00:10.755431" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.0001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.0001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.0001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.0001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..eeca7f754 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.0001-best-head-evaluation/0/report.json @@ -0,0 +1,275 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 4157088, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1100, + "metrics": { + "train": { + "0": { + "precision": 0.9368638960222211, + "recall": 0.9823708173280358, + "f1-score": 0.9590778499977198, + "support": 181971.0 + }, + "1": { + "precision": 0.9042673828707848, + "recall": 0.7155304729745685, + "f1-score": 0.7989032283578745, + "support": 42349.0 + }, + "accuracy": 0.9319944721825963, + "macro avg": { + "precision": 0.9205656394465029, + "recall": 0.8489506451513021, + "f1-score": 0.8789905391777972, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9307100544768834, + "recall": 0.9319944721825963, + "f1-score": 0.9288387493743879, + "support": 224320.0 + }, + "cross-entropy": 0.15546864018533366, + "roc-auc": 0.9714250727707411, + "score": 0.9714250727707411 + }, + "val": { + "0": { + "precision": 0.9298532731376975, + "recall": 0.9842303327160863, + "f1-score": 0.956269405995183, + "support": 16741.0 + }, + "1": { + "precision": 0.890818858560794, + "recall": 0.6340889019723285, + "f1-score": 0.7408426483233018, + "support": 3397.0 + }, + "accuracy": 0.9251663521700269, + "macro avg": { + "precision": 0.9103360658492458, + "recall": 0.8091596173442074, + "f1-score": 0.8485560271592424, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.923268711298501, + "recall": 0.9251663521700269, + "f1-score": 0.9199299137014406, + "support": 20138.0 + }, + "cross-entropy": 0.17668222837030997, + "roc-auc": 0.9580729733437149, + "score": 0.9580729733437149 + }, + "test": { + "0": { + "precision": 0.9272727272727272, + "recall": 0.9775614208564691, + "f1-score": 0.9517532491576258, + "support": 13147.0 + }, + "1": { + "precision": 0.8788501026694046, + "recall": 0.6797966963151207, + "f1-score": 0.7666129321153502, + "support": 3148.0 + }, + "accuracy": 0.9200368211107701, + "macro avg": { + "precision": 0.9030614149710658, + "recall": 0.8286790585857949, + "f1-score": 0.859183090636488, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9179180526945585, + "recall": 0.9200368211107701, + "f1-score": 0.9159863440917109, + "support": 16295.0 + }, + "cross-entropy": 0.18111153198277266, + "roc-auc": 0.9613215034297445, + "score": 0.9613215034297445 + } + }, + "time": "0:10:44.579894", + "diversity": { + "train": { + "mean_centered_corr": 0.02127973735332489, + "mean_pairwise_disagreement": 0.024082001590469835, + "member_std": 0.02185266837477684 + }, + "val": { + "mean_centered_corr": 0.021363921463489532, + "mean_pairwise_disagreement": 0.020554144147318983, + "member_std": 0.022689195349812508 + }, + "test": { + "mean_centered_corr": 0.031094567850232124, + "mean_pairwise_disagreement": 0.026469256352136514, + "member_std": 0.02432551421225071 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 5 + ], + "head_selection_time": "0:00:01.310251" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.0001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.0001-evaluation/0.toml new file mode 100644 index 000000000..63cde1ed1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.0001-evaluation/0.toml @@ -0,0 +1,41 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 8 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.0001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.0001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.0001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.0001-evaluation/0/report.json new file mode 100644 index 000000000..1bcfacc8c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.0001-evaluation/0/report.json @@ -0,0 +1,271 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 4157088, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1100, + "metrics": { + "train": { + "0": { + "precision": 0.9372981517637021, + "recall": 0.9840469085733441, + "f1-score": 0.9601038019612994, + "support": 181971.0 + }, + "1": { + "precision": 0.9127520812670934, + "recall": 0.7171361779498926, + "f1-score": 0.8032054164132131, + "support": 42349.0 + }, + "accuracy": 0.9336572753209701, + "macro avg": { + "precision": 0.9250251165153978, + "recall": 0.8505915432616183, + "f1-score": 0.8816546091872562, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9326641399080455, + "recall": 0.9336572753209701, + "f1-score": 0.9304832165049162, + "support": 224320.0 + }, + "cross-entropy": 0.15236869424446178, + "roc-auc": 0.9730574685406277, + "score": 0.9730574685406277 + }, + "val": { + "0": { + "precision": 0.9290057425965544, + "recall": 0.9856639388328057, + "f1-score": 0.9564965365330551, + "support": 16741.0 + }, + "1": { + "precision": 0.898989898989899, + "recall": 0.6287901089196349, + "f1-score": 0.7399965355967435, + "support": 3397.0 + }, + "accuracy": 0.9254642963551495, + "macro avg": { + "precision": 0.9139978207932267, + "recall": 0.8072270238762203, + "f1-score": 0.8482465360648993, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9239424880165659, + "recall": 0.9254642963551495, + "f1-score": 0.9199760030550209, + "support": 20138.0 + }, + "cross-entropy": 0.17485629796641303, + "roc-auc": 0.9582609925935802, + "score": 0.9582609925935802 + }, + "test": { + "0": { + "precision": 0.9254191144863161, + "recall": 0.9825055145660607, + "f1-score": 0.9531082826046855, + "support": 13147.0 + }, + "1": { + "precision": 0.9015832263585793, + "recall": 0.6693138500635324, + "f1-score": 0.7682771194165907, + "support": 3148.0 + }, + "accuracy": 0.9220006136851795, + "macro avg": { + "precision": 0.9135011704224477, + "recall": 0.8259096823147966, + "f1-score": 0.8606927010106381, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9208143046780242, + "recall": 0.9220006136851795, + "f1-score": 0.9174011023827694, + "support": 16295.0 + }, + "cross-entropy": 0.17807658826679365, + "roc-auc": 0.9623517001429152, + "score": 0.9623517001429152 + } + }, + "time": "0:10:44.579894", + "diversity": { + "train": { + "mean_centered_corr": 0.02127973735332489, + "mean_pairwise_disagreement": 0.024082001590469835, + "member_std": 0.02185266837477684 + }, + "val": { + "mean_centered_corr": 0.021363921463489532, + "mean_pairwise_disagreement": 0.020554144147318983, + "member_std": 0.022689195349812508 + }, + "test": { + "mean_centered_corr": 0.031094567850232124, + "mean_pairwise_disagreement": 0.026469256352136514, + "member_std": 0.02432551421225071 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.0001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.0001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.0001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.0001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..011b118b1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.0001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,280 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 4157088, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1100, + "metrics": { + "train": { + "0": { + "precision": 0.9338260534883963, + "recall": 0.9866572146111193, + "f1-score": 0.9595149597714817, + "support": 181971.0 + }, + "1": { + "precision": 0.924252823360579, + "recall": 0.6995678764551702, + "f1-score": 0.7963657379406744, + "support": 42349.0 + }, + "accuracy": 0.9324580955777461, + "macro avg": { + "precision": 0.9290394384244877, + "recall": 0.8431125455331447, + "f1-score": 0.8779403488560781, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9320187392824275, + "recall": 0.9324580955777461, + "f1-score": 0.9287142893216206, + "support": 224320.0 + }, + "cross-entropy": 0.15364890919870994, + "roc-auc": 0.9726374295788413, + "score": 0.9726374295788413 + }, + "val": { + "0": { + "precision": 0.9273736750602882, + "recall": 0.9877546144196881, + "f1-score": 0.9566122874002083, + "support": 16741.0 + }, + "1": { + "precision": 0.9111400086692675, + "recall": 0.6187812775978805, + "f1-score": 0.7370266479663394, + "support": 3397.0 + }, + "accuracy": 0.9255139537193365, + "macro avg": { + "precision": 0.9192568418647779, + "recall": 0.8032679460087844, + "f1-score": 0.8468194676832739, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9246352817376993, + "recall": 0.9255139537193365, + "f1-score": 0.91957124970248, + "support": 20138.0 + }, + "cross-entropy": 0.17514310049503773, + "roc-auc": 0.9588554974164651, + "score": 0.9588554974164651 + }, + "test": { + "0": { + "precision": 0.9220917822838848, + "recall": 0.9857762227124058, + "f1-score": 0.9528711124182045, + "support": 13147.0 + }, + "1": { + "precision": 0.9165178571428572, + "recall": 0.6521601016518425, + "f1-score": 0.7620638455827765, + "support": 3148.0 + }, + "accuracy": 0.9213255599877263, + "macro avg": { + "precision": 0.919304819713371, + "recall": 0.8189681621821241, + "f1-score": 0.8574674790004906, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9210149663069621, + "recall": 0.9213255599877263, + "f1-score": 0.9160094201200807, + "support": 16295.0 + }, + "cross-entropy": 0.17819981543536048, + "roc-auc": 0.9627966975715613, + "score": 0.9627966975715613 + } + }, + "time": "0:10:44.579894", + "diversity": { + "train": { + "mean_centered_corr": 0.02127973735332489, + "mean_pairwise_disagreement": 0.024082001590469835, + "member_std": 0.02185266837477684 + }, + "val": { + "mean_centered_corr": 0.021363921463489532, + "mean_pairwise_disagreement": 0.020554144147318983, + "member_std": 0.022689195349812508 + }, + "test": { + "mean_centered_corr": 0.031094567850232124, + "mean_pairwise_disagreement": 0.026469256352136514, + "member_std": 0.02432551421225071 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 5, + 31, + 15, + 13, + 19, + 28 + ], + "head_selection_time": "0:00:10.595939" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..d737c8b9f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.001-best-head-evaluation/0/report.json @@ -0,0 +1,275 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 4157088, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9432888064186233, + "recall": 0.9833434997884278, + "f1-score": 0.9628997844856579, + "support": 181971.0 + }, + "1": { + "precision": 0.9124545086938941, + "recall": 0.7459680275803443, + "f1-score": 0.820854607579478, + "support": 42349.0 + }, + "accuracy": 0.9385297788873038, + "macro avg": { + "precision": 0.9278716575562587, + "recall": 0.8646557636843861, + "f1-score": 0.891877196032568, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9374676505950474, + "recall": 0.9385297788873038, + "f1-score": 0.9360833116040611, + "support": 224320.0 + }, + "cross-entropy": 0.14210167519909825, + "roc-auc": 0.9767605177962447, + "score": 0.9767605177962447 + }, + "val": { + "0": { + "precision": 0.9303139823808448, + "recall": 0.9840511319514963, + "f1-score": 0.9564283433481379, + "support": 16741.0 + }, + "1": { + "precision": 0.8901234567901235, + "recall": 0.6367382984986752, + "f1-score": 0.7424060408443452, + "support": 3397.0 + }, + "accuracy": 0.9254642963551495, + "macro avg": { + "precision": 0.9102187195854842, + "recall": 0.8103947152250858, + "f1-score": 0.8494171920962416, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.923534400722702, + "recall": 0.9254642963551495, + "f1-score": 0.9203257630717756, + "support": 20138.0 + }, + "cross-entropy": 0.1861201295625259, + "roc-auc": 0.9565663135937417, + "score": 0.9565663135937417 + }, + "test": { + "0": { + "precision": 0.9246846846846847, + "recall": 0.9758880352932228, + "f1-score": 0.9495966249722448, + "support": 13147.0 + }, + "1": { + "precision": 0.8690082644628099, + "recall": 0.6680432020330368, + "f1-score": 0.7553879310344828, + "support": 3148.0 + }, + "accuracy": 0.9164160785517029, + "macro avg": { + "precision": 0.8968464745737472, + "recall": 0.8219656186631298, + "f1-score": 0.8524922780033638, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9139286631530208, + "recall": 0.9164160785517029, + "f1-score": 0.9120778174536149, + "support": 16295.0 + }, + "cross-entropy": 0.19882303213503738, + "roc-auc": 0.9586702567362371, + "score": 0.9586702567362371 + } + }, + "time": "0:12:11.920322", + "diversity": { + "train": { + "mean_centered_corr": 0.0516890287399292, + "mean_pairwise_disagreement": 0.030828574027311213, + "member_std": 0.028233278542757034 + }, + "val": { + "mean_centered_corr": 0.061428945511579514, + "mean_pairwise_disagreement": 0.028893978323759607, + "member_std": 0.030158301815390587 + }, + "test": { + "mean_centered_corr": 0.0725783258676529, + "mean_pairwise_disagreement": 0.03697737778261687, + "member_std": 0.032935526221990585 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21 + ], + "head_selection_time": "0:00:00.906547" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.001-evaluation/0.toml new file mode 100644 index 000000000..7f4852bd8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.001-evaluation/0.toml @@ -0,0 +1,41 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 8 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.001-evaluation/0/report.json new file mode 100644 index 000000000..5ffb781a1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.001-evaluation/0/report.json @@ -0,0 +1,271 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 4157088, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9477062667879275, + "recall": 0.9849591418412824, + "f1-score": 0.9659736725097346, + "support": 181971.0 + }, + "1": { + "precision": 0.9222354813046937, + "recall": 0.7664643793241871, + "f1-score": 0.8371655167966987, + "support": 42349.0 + }, + "accuracy": 0.9437098787446505, + "macro avg": { + "precision": 0.9349708740463105, + "recall": 0.8757117605827347, + "f1-score": 0.9015695946532167, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9428976795267405, + "recall": 0.9437098787446505, + "f1-score": 0.941656194860433, + "support": 224320.0 + }, + "cross-entropy": 0.13317658377318187, + "roc-auc": 0.9805624896867443, + "score": 0.9805624896867443 + }, + "val": { + "0": { + "precision": 0.935017639695004, + "recall": 0.9815423212472373, + "f1-score": 0.957715284860848, + "support": 16741.0 + }, + "1": { + "precision": 0.8794851794071763, + "recall": 0.6638210185457757, + "f1-score": 0.7565844656936755, + "support": 3397.0 + }, + "accuracy": 0.9279471645645049, + "macro avg": { + "precision": 0.9072514095510902, + "recall": 0.8226816698965065, + "f1-score": 0.8571498752772617, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.925650087425774, + "recall": 0.9279471645645049, + "f1-score": 0.923787318195296, + "support": 20138.0 + }, + "cross-entropy": 0.17461587348635396, + "roc-auc": 0.9580761824634811, + "score": 0.9580761824634811 + }, + "test": { + "0": { + "precision": 0.9301449275362319, + "recall": 0.9763444131741081, + "f1-score": 0.9526848999888671, + "support": 13147.0 + }, + "1": { + "precision": 0.8753507014028056, + "recall": 0.6937738246505718, + "f1-score": 0.7740563530037214, + "support": 3148.0 + }, + "accuracy": 0.9217551396133783, + "macro avg": { + "precision": 0.9027478144695187, + "recall": 0.83505911891234, + "f1-score": 0.8633706264962943, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9195593353995012, + "recall": 0.9217551396133783, + "f1-score": 0.9181759913721602, + "support": 16295.0 + }, + "cross-entropy": 0.17887228382743733, + "roc-auc": 0.962025146401907, + "score": 0.962025146401907 + } + }, + "time": "0:12:11.920322", + "diversity": { + "train": { + "mean_centered_corr": 0.0516890287399292, + "mean_pairwise_disagreement": 0.030828574027311213, + "member_std": 0.028233278542757034 + }, + "val": { + "mean_centered_corr": 0.061428945511579514, + "mean_pairwise_disagreement": 0.028893978323759607, + "member_std": 0.030158301815390587 + }, + "test": { + "mean_centered_corr": 0.0725783258676529, + "mean_pairwise_disagreement": 0.03697737778261687, + "member_std": 0.032935526221990585 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..3c4449fab --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,282 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 4157088, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9449847896337933, + "recall": 0.9866901868979123, + "f1-score": 0.9653872727321606, + "support": 181971.0 + }, + "1": { + "precision": 0.9294247916545253, + "recall": 0.7531700866608421, + "f1-score": 0.8320659475393585, + "support": 42349.0 + }, + "accuracy": 0.9426043152639088, + "macro avg": { + "precision": 0.9372047906441593, + "recall": 0.8699301367793772, + "f1-score": 0.8987266101357596, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9420472434746276, + "recall": 0.9426043152639088, + "f1-score": 0.9402177613172623, + "support": 224320.0 + }, + "cross-entropy": 0.13407185107135744, + "roc-auc": 0.9801324950911569, + "score": 0.9801324950911569 + }, + "val": { + "0": { + "precision": 0.9316949152542373, + "recall": 0.9850666029508393, + "f1-score": 0.9576376992537964, + "support": 16741.0 + }, + "1": { + "precision": 0.8974569319114027, + "recall": 0.644097733294083, + "f1-score": 0.7499571550985433, + "support": 3397.0 + }, + "accuracy": 0.927549905651008, + "macro avg": { + "precision": 0.91457592358282, + "recall": 0.8145821681224612, + "f1-score": 0.8537974271761699, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9259194445314441, + "recall": 0.927549905651008, + "f1-score": 0.9226048852456827, + "support": 20138.0 + }, + "cross-entropy": 0.1751815095057731, + "roc-auc": 0.9588770556676071, + "score": 0.9588770556676071 + }, + "test": { + "0": { + "precision": 0.926901215914814, + "recall": 0.9799193732410436, + "f1-score": 0.9526732233971752, + "support": 13147.0 + }, + "1": { + "precision": 0.8898163606010017, + "recall": 0.6772554002541296, + "f1-score": 0.7691197691197691, + "support": 3148.0 + }, + "accuracy": 0.9214482970236268, + "macro avg": { + "precision": 0.9083587882579078, + "recall": 0.8285873867475866, + "f1-score": 0.8608964962584722, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9197368633816517, + "recall": 0.9214482970236268, + "f1-score": 0.9172128813250504, + "support": 16295.0 + }, + "cross-entropy": 0.180176006712123, + "roc-auc": 0.9621469244895637, + "score": 0.9621469244895637 + } + }, + "time": "0:12:11.920322", + "diversity": { + "train": { + "mean_centered_corr": 0.0516890287399292, + "mean_pairwise_disagreement": 0.030828574027311213, + "member_std": 0.028233278542757034 + }, + "val": { + "mean_centered_corr": 0.061428945511579514, + "mean_pairwise_disagreement": 0.028893978323759607, + "member_std": 0.030158301815390587 + }, + "test": { + "mean_centered_corr": 0.0725783258676529, + "mean_pairwise_disagreement": 0.03697737778261687, + "member_std": 0.032935526221990585 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21, + 24, + 31, + 12, + 8, + 30, + 24, + 9 + ], + "head_selection_time": "0:00:09.478138" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise1e-05-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise1e-05-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise1e-05-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise1e-05-best-head-evaluation/0/report.json new file mode 100644 index 000000000..db2179a27 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise1e-05-best-head-evaluation/0/report.json @@ -0,0 +1,275 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 4157088, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9402544726609582, + "recall": 0.9860087596375247, + "f1-score": 0.9625882182528293, + "support": 181971.0 + }, + "1": { + "precision": 0.9239863856213053, + "recall": 0.7307846702401474, + "f1-score": 0.816106957794391, + "support": 42349.0 + }, + "accuracy": 0.9378254279600571, + "macro avg": { + "precision": 0.9321204291411318, + "recall": 0.8583967149388361, + "f1-score": 0.8893475880236101, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.937183247544864, + "recall": 0.9378254279600571, + "f1-score": 0.9349342645297801, + "support": 224320.0 + }, + "cross-entropy": 0.14254385707563483, + "roc-auc": 0.9770372580348661, + "score": 0.9770372580348661 + }, + "val": { + "0": { + "precision": 0.9269017321598745, + "recall": 0.9876948808314916, + "f1-score": 0.9563331405436668, + "support": 16741.0 + }, + "1": { + "precision": 0.9103958242714224, + "recall": 0.6161318810715337, + "f1-score": 0.7349016853932584, + "support": 3397.0 + }, + "accuracy": 0.9250173800774655, + "macro avg": { + "precision": 0.9186487782156485, + "recall": 0.8019133809515127, + "f1-score": 0.8456174129684626, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9241174154900427, + "recall": 0.9250173800774655, + "f1-score": 0.9189807394538894, + "support": 20138.0 + }, + "cross-entropy": 0.19012849944818658, + "roc-auc": 0.9560226359526883, + "score": 0.9560226359526883 + }, + "test": { + "0": { + "precision": 0.9207786651454649, + "recall": 0.9822012626454705, + "f1-score": 0.9504986934599389, + "support": 13147.0 + }, + "1": { + "precision": 0.8969616908850726, + "recall": 0.6470775095298602, + "f1-score": 0.7517992249492527, + "support": 3148.0 + }, + "accuracy": 0.917459343356858, + "macro avg": { + "precision": 0.9088701780152688, + "recall": 0.8146393860876653, + "f1-score": 0.8511489592045958, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9161775092711651, + "recall": 0.917459343356858, + "f1-score": 0.9121123217587029, + "support": 16295.0 + }, + "cross-entropy": 0.2007697564863984, + "roc-auc": 0.9594298354768371, + "score": 0.9594298354768371 + } + }, + "time": "0:12:13.002733", + "diversity": { + "train": { + "mean_centered_corr": 0.05281975865364075, + "mean_pairwise_disagreement": 0.030102508728889605, + "member_std": 0.02849932573735714 + }, + "val": { + "mean_centered_corr": 0.059580858796834946, + "mean_pairwise_disagreement": 0.027920053245509214, + "member_std": 0.030231283977627754 + }, + "test": { + "mean_centered_corr": 0.07347472012042999, + "mean_pairwise_disagreement": 0.03518185372516802, + "member_std": 0.03243069350719452 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21 + ], + "head_selection_time": "0:00:00.973893" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise1e-05-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise1e-05-evaluation/0.toml new file mode 100644 index 000000000..949a0af96 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise1e-05-evaluation/0.toml @@ -0,0 +1,41 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 8 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise1e-05-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise1e-05-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise1e-05-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise1e-05-evaluation/0/report.json new file mode 100644 index 000000000..40cdb12ad --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise1e-05-evaluation/0/report.json @@ -0,0 +1,271 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 4157088, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9450154648936394, + "recall": 0.9872726972979211, + "f1-score": 0.9656820191410964, + "support": 181971.0 + }, + "1": { + "precision": 0.9323044545773412, + "recall": 0.7531700866608421, + "f1-score": 0.8332179569232376, + "support": 42349.0 + }, + "accuracy": 0.9430768544935806, + "macro avg": { + "precision": 0.9386599597354903, + "recall": 0.8702213919793815, + "f1-score": 0.899449988032167, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9426157743806003, + "recall": 0.9430768544935806, + "f1-score": 0.9406743489785425, + "support": 224320.0 + }, + "cross-entropy": 0.1326793673321411, + "roc-auc": 0.9809027579015627, + "score": 0.9809027579015627 + }, + "val": { + "0": { + "precision": 0.9323176257362936, + "recall": 0.98327459530494, + "f1-score": 0.9571183533447685, + "support": 16741.0 + }, + "1": { + "precision": 0.887187751813054, + "recall": 0.6482190167795113, + "f1-score": 0.7491069909848613, + "support": 3397.0 + }, + "accuracy": 0.9267553878240143, + "macro avg": { + "precision": 0.9097526887746739, + "recall": 0.8157468060422257, + "f1-score": 0.8531126721648149, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9247048448386253, + "recall": 0.9267553878240143, + "f1-score": 0.9220297349151029, + "support": 20138.0 + }, + "cross-entropy": 0.17484437515862924, + "roc-auc": 0.958187182838957, + "score": 0.958187182838957 + }, + "test": { + "0": { + "precision": 0.927964270278058, + "recall": 0.979843310260896, + "f1-score": 0.9531984165155943, + "support": 13147.0 + }, + "1": { + "precision": 0.8901782014090344, + "recall": 0.6823379923761118, + "f1-score": 0.7725229275310196, + "support": 3148.0 + }, + "accuracy": 0.9223688247928813, + "macro avg": { + "precision": 0.9090712358435462, + "recall": 0.8310906513185039, + "f1-score": 0.862860672023307, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9206644516343214, + "recall": 0.9223688247928813, + "f1-score": 0.9182940630744504, + "support": 16295.0 + }, + "cross-entropy": 0.17826501548417395, + "roc-auc": 0.9624786972914716, + "score": 0.9624786972914716 + } + }, + "time": "0:12:13.002733", + "diversity": { + "train": { + "mean_centered_corr": 0.05281975865364075, + "mean_pairwise_disagreement": 0.030102508728889605, + "member_std": 0.02849932573735714 + }, + "val": { + "mean_centered_corr": 0.059580858796834946, + "mean_pairwise_disagreement": 0.027920053245509214, + "member_std": 0.030231283977627754 + }, + "test": { + "mean_centered_corr": 0.07347472012042999, + "mean_pairwise_disagreement": 0.03518185372516802, + "member_std": 0.03243069350719452 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise1e-05-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise1e-05-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise1e-05-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise1e-05-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..8c051bd05 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise1e-05-greedy-heads-evaluation/0/report.json @@ -0,0 +1,287 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 4157088, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9441635038952028, + "recall": 0.9870363959092383, + "f1-score": 0.965124057978802, + "support": 181971.0 + }, + "1": { + "precision": 0.9307927008155841, + "recall": 0.7491794375309925, + "f1-score": 0.8301694250016354, + "support": 42349.0 + }, + "accuracy": 0.9421317760342368, + "macro avg": { + "precision": 0.9374781023553934, + "recall": 0.8681079167201153, + "f1-score": 0.8976467414902187, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9416392522028937, + "recall": 0.9421317760342368, + "f1-score": 0.9396461971017066, + "support": 224320.0 + }, + "cross-entropy": 0.13438484231859613, + "roc-auc": 0.9800307814893711, + "score": 0.9800307814893711 + }, + "val": { + "0": { + "precision": 0.931937765205092, + "recall": 0.9839316647751031, + "f1-score": 0.9572291957229195, + "support": 16741.0 + }, + "1": { + "precision": 0.8907835972391392, + "recall": 0.6458639976449808, + "f1-score": 0.7488054607508532, + "support": 3397.0 + }, + "accuracy": 0.9269043599165756, + "macro avg": { + "precision": 0.9113606812221156, + "recall": 0.8148978312100419, + "f1-score": 0.8530173282368864, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.924995630505502, + "recall": 0.9269043599165756, + "f1-score": 0.9220710157795234, + "support": 20138.0 + }, + "cross-entropy": 0.17479880731904499, + "roc-auc": 0.9588073342436448, + "score": 0.9588073342436448 + }, + "test": { + "0": { + "precision": 0.9273630408177957, + "recall": 0.979843310260896, + "f1-score": 0.952881130261114, + "support": 13147.0 + }, + "1": { + "precision": 0.8897670549084858, + "recall": 0.6794790343074968, + "f1-score": 0.7705331412103746, + "support": 3148.0 + }, + "accuracy": 0.9218165081313286, + "macro avg": { + "precision": 0.9085650478631409, + "recall": 0.8296611722841964, + "f1-score": 0.8617071357357443, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9200999439388448, + "recall": 0.9218165081313286, + "f1-score": 0.917653669719124, + "support": 16295.0 + }, + "cross-entropy": 0.17874164813587776, + "roc-auc": 0.9625660923992208, + "score": 0.9625660923992208 + } + }, + "time": "0:12:13.002733", + "diversity": { + "train": { + "mean_centered_corr": 0.05281975865364075, + "mean_pairwise_disagreement": 0.030102508728889605, + "member_std": 0.02849932573735714 + }, + "val": { + "mean_centered_corr": 0.059580858796834946, + "mean_pairwise_disagreement": 0.027920053245509214, + "member_std": 0.030231283977627754 + }, + "test": { + "mean_centered_corr": 0.07347472012042999, + "mean_pairwise_disagreement": 0.03518185372516802, + "member_std": 0.03243069350719452 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21, + 31, + 24, + 30, + 6, + 7, + 0, + 9, + 2, + 30, + 24, + 31, + 11 + ], + "head_selection_time": "0:00:13.085238" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_four_combined-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_four_combined-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_four_combined-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_four_combined-best-head-evaluation/0/report.json new file mode 100644 index 000000000..f5cc9f8c2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_four_combined-best-head-evaluation/0/report.json @@ -0,0 +1,702 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_four_combined-evaluation/_esam_diagnostics/all_four_combined-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.238513825213292, + "mae": 0.12740404903888702, + "r2": 0.6365530490875244, + "score": -0.238513825213292 + }, + "val": { + "rmse": 0.2376718937442043, + "mae": 0.13975045084953308, + "r2": 0.6226825714111328, + "score": -0.2376718937442043 + }, + "test": { + "rmse": 0.2539783962159738, + "mae": 0.17701080441474915, + "r2": 0.5592085719108582, + "score": -0.2539783962159738 + } + }, + "time": "0:03:25.619596", + "diversity": { + "train": { + "mean_centered_corr": -0.018900908529758453, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04601670801639557 + }, + "val": { + "mean_centered_corr": -0.01799248345196247, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05731586739420891 + }, + "test": { + "mean_centered_corr": -0.019703488796949387, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07458258420228958 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25 + ], + "head_selection_time": "0:00:00.015338" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_four_combined-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_four_combined-evaluation/0.toml new file mode 100644 index 000000000..aaa491b98 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_four_combined-evaluation/0.toml @@ -0,0 +1,75 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_four_combined-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_four_combined-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_four_combined-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_four_combined-evaluation/0/report.json new file mode 100644 index 000000000..624449948 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_four_combined-evaluation/0/report.json @@ -0,0 +1,698 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_four_combined-evaluation/_esam_diagnostics/all_four_combined-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2331321585964892, + "mae": 0.12105797231197357, + "r2": 0.6527692079544067, + "score": -0.2331321585964892 + }, + "val": { + "rmse": 0.2348792849855834, + "mae": 0.12763135135173798, + "r2": 0.6314972639083862, + "score": -0.2348792849855834 + }, + "test": { + "rmse": 0.2326272249658597, + "mae": 0.1447165459394455, + "r2": 0.6302052736282349, + "score": -0.2326272249658597 + } + }, + "time": "0:03:25.619596", + "diversity": { + "train": { + "mean_centered_corr": -0.018900908529758453, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04601670801639557 + }, + "val": { + "mean_centered_corr": -0.01799248345196247, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05731586739420891 + }, + "test": { + "mean_centered_corr": -0.019703488796949387, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07458258420228958 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_four_combined-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_four_combined-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_four_combined-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_four_combined-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..94589782c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_four_combined-greedy-heads-evaluation/0/report.json @@ -0,0 +1,710 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_four_combined-evaluation/_esam_diagnostics/all_four_combined-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23288428364808966, + "mae": 0.12141954898834229, + "r2": 0.6535071730613708, + "score": -0.23288428364808966 + }, + "val": { + "rmse": 0.23335850745168515, + "mae": 0.1262979656457901, + "r2": 0.6362537145614624, + "score": -0.23335850745168515 + }, + "test": { + "rmse": 0.23263890687901315, + "mae": 0.14496733248233795, + "r2": 0.6301681995391846, + "score": -0.23263890687901315 + } + }, + "time": "0:03:25.619596", + "diversity": { + "train": { + "mean_centered_corr": -0.018900908529758453, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04601670801639557 + }, + "val": { + "mean_centered_corr": -0.01799248345196247, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05731586739420891 + }, + "test": { + "mean_centered_corr": -0.019703488796949387, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07458258420228958 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25, + 10, + 19, + 8, + 12, + 28, + 21, + 24, + 20 + ], + "head_selection_time": "0:00:00.147069" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_cf_fisd-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_cf_fisd-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_cf_fisd-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_cf_fisd-best-head-evaluation/0/report.json new file mode 100644 index 000000000..9c84f2134 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_cf_fisd-best-head-evaluation/0/report.json @@ -0,0 +1,249 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_cf_fisd-evaluation/_esam_diagnostics/all_minus_cf_fisd-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23669818404848408, + "mae": 0.1265603005886078, + "r2": 0.6420653462409973, + "score": -0.23669818404848408 + }, + "val": { + "rmse": 0.23808479027162685, + "mae": 0.13432005047798157, + "r2": 0.6213704347610474, + "score": -0.23808479027162685 + }, + "test": { + "rmse": 0.27950919024280946, + "mae": 0.20337718725204468, + "r2": 0.46613460779190063, + "score": -0.27950919024280946 + } + }, + "time": "0:02:54.620098", + "diversity": { + "train": { + "mean_centered_corr": -0.018507609143853188, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046238355338573456 + }, + "val": { + "mean_centered_corr": -0.019183674827218056, + "mean_pairwise_disagreement": NaN, + "member_std": 0.059328801929950714 + }, + "test": { + "mean_centered_corr": -0.020014170557260513, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07509658485651016 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25 + ], + "head_selection_time": "0:00:00.015242" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_cf_fisd-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_cf_fisd-evaluation/0.toml new file mode 100644 index 000000000..f1eb43eee --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_cf_fisd-evaluation/0.toml @@ -0,0 +1,64 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_cf_fisd-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_cf_fisd-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_cf_fisd-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_cf_fisd-evaluation/0/report.json new file mode 100644 index 000000000..b1b517b94 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_cf_fisd-evaluation/0/report.json @@ -0,0 +1,245 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_cf_fisd-evaluation/_esam_diagnostics/all_minus_cf_fisd-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2320688756626341, + "mae": 0.1213240846991539, + "r2": 0.6559293270111084, + "score": -0.2320688756626341 + }, + "val": { + "rmse": 0.23469635866718716, + "mae": 0.12645497918128967, + "r2": 0.6320710182189941, + "score": -0.23469635866718716 + }, + "test": { + "rmse": 0.23237236681744675, + "mae": 0.14428266882896423, + "r2": 0.6310151219367981, + "score": -0.23237236681744675 + } + }, + "time": "0:02:54.620098", + "diversity": { + "train": { + "mean_centered_corr": -0.018507609143853188, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046238355338573456 + }, + "val": { + "mean_centered_corr": -0.019183674827218056, + "mean_pairwise_disagreement": NaN, + "member_std": 0.059328801929950714 + }, + "test": { + "mean_centered_corr": -0.020014170557260513, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07509658485651016 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_cf_fisd-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_cf_fisd-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_cf_fisd-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_cf_fisd-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..1120f9b75 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_cf_fisd-greedy-heads-evaluation/0/report.json @@ -0,0 +1,256 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_cf_fisd-evaluation/_esam_diagnostics/all_minus_cf_fisd-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23281312116757674, + "mae": 0.12442371994256973, + "r2": 0.653718888759613, + "score": -0.23281312116757674 + }, + "val": { + "rmse": 0.23321538793887137, + "mae": 0.12674209475517273, + "r2": 0.6366997957229614, + "score": -0.23321538793887137 + }, + "test": { + "rmse": 0.23519795893846665, + "mae": 0.1507532298564911, + "r2": 0.6219870448112488, + "score": -0.23519795893846665 + } + }, + "time": "0:02:54.620098", + "diversity": { + "train": { + "mean_centered_corr": -0.018507609143853188, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046238355338573456 + }, + "val": { + "mean_centered_corr": -0.019183674827218056, + "mean_pairwise_disagreement": NaN, + "member_std": 0.059328801929950714 + }, + "test": { + "mean_centered_corr": -0.020014170557260513, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07509658485651016 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25, + 24, + 19, + 4, + 12, + 24, + 31, + 18 + ], + "head_selection_time": "0:00:00.130392" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_esam-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_esam-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_esam-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_esam-best-head-evaluation/0/report.json new file mode 100644 index 000000000..f98d1d253 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_esam-best-head-evaluation/0/report.json @@ -0,0 +1,692 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23559077756802618, + "mae": 0.12391819804906845, + "r2": 0.6454067230224609, + "score": -0.23559077756802618 + }, + "val": { + "rmse": 0.23762339316571812, + "mae": 0.13952432572841644, + "r2": 0.6228364706039429, + "score": -0.23762339316571812 + }, + "test": { + "rmse": 0.2548487995005296, + "mae": 0.1803659200668335, + "r2": 0.5561821460723877, + "score": -0.2548487995005296 + } + }, + "time": "0:02:05.501890", + "diversity": { + "train": { + "mean_centered_corr": -0.019058862701058388, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04700104147195816 + }, + "val": { + "mean_centered_corr": -0.01848544180393219, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06533127278089523 + }, + "test": { + "mean_centered_corr": -0.01928732916712761, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08419054746627808 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 25 + ], + "head_selection_time": "0:00:00.015176" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_esam-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_esam-evaluation/0.toml new file mode 100644 index 000000000..1ee60827a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_esam-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_esam-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_esam-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_esam-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_esam-evaluation/0/report.json new file mode 100644 index 000000000..f5a72f4b6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_esam-evaluation/0/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23240700041031828, + "mae": 0.11997766047716141, + "r2": 0.6549259424209595, + "score": -0.23240700041031828 + }, + "val": { + "rmse": 0.23494022877441162, + "mae": 0.1266379952430725, + "r2": 0.6313060522079468, + "score": -0.23494022877441162 + }, + "test": { + "rmse": 0.2319230328213383, + "mae": 0.14268861711025238, + "r2": 0.6324407458305359, + "score": -0.2319230328213383 + } + }, + "time": "0:02:05.501890", + "diversity": { + "train": { + "mean_centered_corr": -0.019058862701058388, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04700104147195816 + }, + "val": { + "mean_centered_corr": -0.01848544180393219, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06533127278089523 + }, + "test": { + "mean_centered_corr": -0.01928732916712761, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08419054746627808 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_esam-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_esam-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_esam-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_esam-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..4150daf8f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_esam-greedy-heads-evaluation/0/report.json @@ -0,0 +1,697 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23248251756958926, + "mae": 0.12231885641813278, + "r2": 0.6547016501426697, + "score": -0.23248251756958926 + }, + "val": { + "rmse": 0.23366970594362974, + "mae": 0.1270563155412674, + "r2": 0.6352829337120056, + "score": -0.23366970594362974 + }, + "test": { + "rmse": 0.23132655964186158, + "mae": 0.14191654324531555, + "r2": 0.6343289613723755, + "score": -0.23132655964186158 + } + }, + "time": "0:02:05.501890", + "diversity": { + "train": { + "mean_centered_corr": -0.019058862701058388, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04700104147195816 + }, + "val": { + "mean_centered_corr": -0.01848544180393219, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06533127278089523 + }, + "test": { + "mean_centered_corr": -0.01928732916712761, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08419054746627808 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 25, + 21, + 19, + 0, + 22, + 3 + ], + "head_selection_time": "0:00:00.101481" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_mfb-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_mfb-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_mfb-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_mfb-best-head-evaluation/0/report.json new file mode 100644 index 000000000..5dabb8889 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_mfb-best-head-evaluation/0/report.json @@ -0,0 +1,666 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_mfb-evaluation/_esam_diagnostics/all_minus_mfb-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23953673006222745, + "mae": 0.1305936723947525, + "r2": 0.6334289908409119, + "score": -0.23953673006222745 + }, + "val": { + "rmse": 0.23769862439897374, + "mae": 0.13297532498836517, + "r2": 0.6225976347923279, + "score": -0.23769862439897374 + }, + "test": { + "rmse": 0.24291600363173604, + "mae": 0.1576826125383377, + "r2": 0.5967708826065063, + "score": -0.24291600363173604 + } + }, + "time": "0:03:16.948884", + "diversity": { + "train": { + "mean_centered_corr": -0.02016131393611431, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04428550973534584 + }, + "val": { + "mean_centered_corr": -0.01863192766904831, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06180477887392044 + }, + "test": { + "mean_centered_corr": -0.011988693848252296, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08672565221786499 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0 + ], + "head_selection_time": "0:00:00.015532" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_mfb-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_mfb-evaluation/0.toml new file mode 100644 index 000000000..d4514eeae --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_mfb-evaluation/0.toml @@ -0,0 +1,62 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_mfb-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_mfb-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_mfb-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_mfb-evaluation/0/report.json new file mode 100644 index 000000000..8fdfb8b08 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_mfb-evaluation/0/report.json @@ -0,0 +1,662 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_mfb-evaluation/_esam_diagnostics/all_minus_mfb-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23349763844998986, + "mae": 0.12047048658132553, + "r2": 0.6516796350479126, + "score": -0.23349763844998986 + }, + "val": { + "rmse": 0.23539867958344296, + "mae": 0.12546104192733765, + "r2": 0.6298657655715942, + "score": -0.23539867958344296 + }, + "test": { + "rmse": 0.2332565482846656, + "mae": 0.1435696929693222, + "r2": 0.6282018423080444, + "score": -0.2332565482846656 + } + }, + "time": "0:03:16.948884", + "diversity": { + "train": { + "mean_centered_corr": -0.02016131393611431, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04428550973534584 + }, + "val": { + "mean_centered_corr": -0.01863192766904831, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06180477887392044 + }, + "test": { + "mean_centered_corr": -0.011988693848252296, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08672565221786499 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_mfb-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_mfb-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_mfb-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_mfb-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..3b701aadc --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_mfb-greedy-heads-evaluation/0/report.json @@ -0,0 +1,673 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_mfb-evaluation/_esam_diagnostics/all_minus_mfb-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2332399859921054, + "mae": 0.12097182869911194, + "r2": 0.6524479389190674, + "score": -0.2332399859921054 + }, + "val": { + "rmse": 0.23349088968550719, + "mae": 0.12680040299892426, + "r2": 0.635840892791748, + "score": -0.23349088968550719 + }, + "test": { + "rmse": 0.23959467004131907, + "mae": 0.15605996549129486, + "r2": 0.6077220439910889, + "score": -0.23959467004131907 + } + }, + "time": "0:03:16.948884", + "diversity": { + "train": { + "mean_centered_corr": -0.02016131393611431, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04428550973534584 + }, + "val": { + "mean_centered_corr": -0.01863192766904831, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06180477887392044 + }, + "test": { + "mean_centered_corr": -0.011988693848252296, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08672565221786499 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0, + 12, + 28, + 16, + 19, + 25, + 10, + 20 + ], + "head_selection_time": "0:00:00.131053" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_rla-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_rla-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_rla-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_rla-best-head-evaluation/0/report.json new file mode 100644 index 000000000..fc5abdc56 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_rla-best-head-evaluation/0/report.json @@ -0,0 +1,689 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_rla-evaluation/_esam_diagnostics/all_minus_rla-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2385659861695932, + "mae": 0.12279489636421204, + "r2": 0.6363940834999084, + "score": -0.2385659861695932 + }, + "val": { + "rmse": 0.2383111872393019, + "mae": 0.1314251869916916, + "r2": 0.6206499934196472, + "score": -0.2383111872393019 + }, + "test": { + "rmse": 0.2360172512727007, + "mae": 0.14784888923168182, + "r2": 0.6193488836288452, + "score": -0.2360172512727007 + } + }, + "time": "0:02:10.492471", + "diversity": { + "train": { + "mean_centered_corr": -0.019148705527186394, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04757876694202423 + }, + "val": { + "mean_centered_corr": -0.01928521692752838, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06040926277637482 + }, + "test": { + "mean_centered_corr": -0.019908146932721138, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07882117480039597 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 28 + ], + "head_selection_time": "0:00:00.015224" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_rla-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_rla-evaluation/0.toml new file mode 100644 index 000000000..d5a6282f3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_rla-evaluation/0.toml @@ -0,0 +1,68 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_rla-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_rla-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_rla-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_rla-evaluation/0/report.json new file mode 100644 index 000000000..e3195bd55 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_rla-evaluation/0/report.json @@ -0,0 +1,685 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_rla-evaluation/_esam_diagnostics/all_minus_rla-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23440638967758307, + "mae": 0.12211208790540695, + "r2": 0.6489630937576294, + "score": -0.23440638967758307 + }, + "val": { + "rmse": 0.23564260539703746, + "mae": 0.12916921079158783, + "r2": 0.6290982365608215, + "score": -0.23564260539703746 + }, + "test": { + "rmse": 0.2337545527178023, + "mae": 0.14542868733406067, + "r2": 0.6266125440597534, + "score": -0.2337545527178023 + } + }, + "time": "0:02:10.492471", + "diversity": { + "train": { + "mean_centered_corr": -0.019148705527186394, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04757876694202423 + }, + "val": { + "mean_centered_corr": -0.01928521692752838, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06040926277637482 + }, + "test": { + "mean_centered_corr": -0.019908146932721138, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07882117480039597 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_rla-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_rla-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_rla-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_rla-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..732541603 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_rla-greedy-heads-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_rla-evaluation/_esam_diagnostics/all_minus_rla-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2332182711566056, + "mae": 0.1229761615395546, + "r2": 0.6525126695632935, + "score": -0.2332182711566056 + }, + "val": { + "rmse": 0.23419188657775022, + "mae": 0.1297776848077774, + "r2": 0.6336510181427002, + "score": -0.23419188657775022 + }, + "test": { + "rmse": 0.23442550750747457, + "mae": 0.14977632462978363, + "r2": 0.6244659423828125, + "score": -0.23442550750747457 + } + }, + "time": "0:02:10.492471", + "diversity": { + "train": { + "mean_centered_corr": -0.019148705527186394, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04757876694202423 + }, + "val": { + "mean_centered_corr": -0.01928521692752838, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06040926277637482 + }, + "test": { + "mean_centered_corr": -0.019908146932721138, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07882117480039597 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 28, + 19, + 25, + 12, + 24, + 22 + ], + "head_selection_time": "0:00:00.101853" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.05-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.05-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.05-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.05-best-head-evaluation/0/report.json new file mode 100644 index 000000000..5c112e278 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.05-best-head-evaluation/0/report.json @@ -0,0 +1,643 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23931750895741843, + "mae": 0.13275523483753204, + "r2": 0.6340996623039246, + "score": -0.23931750895741843 + }, + "val": { + "rmse": 0.2394238965987828, + "mae": 0.13270841538906097, + "r2": 0.6170992255210876, + "score": -0.2394238965987828 + }, + "test": { + "rmse": 0.23525657161407812, + "mae": 0.14720264077186584, + "r2": 0.6217986345291138, + "score": -0.23525657161407812 + } + }, + "time": "0:01:03.360837", + "diversity": { + "train": { + "mean_centered_corr": -0.019119493663311005, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046866077929735184 + }, + "val": { + "mean_centered_corr": -0.02003737911581993, + "mean_pairwise_disagreement": NaN, + "member_std": 0.058194298297166824 + }, + "test": { + "mean_centered_corr": -0.01959315314888954, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07949266582727432 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 24 + ], + "head_selection_time": "0:00:00.015297" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.05-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.05-evaluation/0.toml new file mode 100644 index 000000000..2692a646a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.05-evaluation/0.toml @@ -0,0 +1,45 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.05 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.05-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.05-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.05-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.05-evaluation/0/report.json new file mode 100644 index 000000000..40a4a148f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.05-evaluation/0/report.json @@ -0,0 +1,639 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23390890506009623, + "mae": 0.12171443551778793, + "r2": 0.6504515409469604, + "score": -0.23390890506009623 + }, + "val": { + "rmse": 0.2359593327633197, + "mae": 0.12757939100265503, + "r2": 0.6281005144119263, + "score": -0.2359593327633197 + }, + "test": { + "rmse": 0.23536697907032925, + "mae": 0.1493176817893982, + "r2": 0.621443510055542, + "score": -0.23536697907032925 + } + }, + "time": "0:01:03.360837", + "diversity": { + "train": { + "mean_centered_corr": -0.019119493663311005, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046866077929735184 + }, + "val": { + "mean_centered_corr": -0.02003737911581993, + "mean_pairwise_disagreement": NaN, + "member_std": 0.058194298297166824 + }, + "test": { + "mean_centered_corr": -0.01959315314888954, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07949266582727432 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.05-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.05-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.05-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.05-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..ffd05ad90 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.05-greedy-heads-evaluation/0/report.json @@ -0,0 +1,647 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23536915535287303, + "mae": 0.12580418586730957, + "r2": 0.646073579788208, + "score": -0.23536915535287303 + }, + "val": { + "rmse": 0.23452905519184072, + "mae": 0.1281842142343521, + "r2": 0.632595419883728, + "score": -0.23452905519184072 + }, + "test": { + "rmse": 0.23270925018080982, + "mae": 0.14333613216876984, + "r2": 0.6299444437026978, + "score": -0.23270925018080982 + } + }, + "time": "0:01:03.360837", + "diversity": { + "train": { + "mean_centered_corr": -0.019119493663311005, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046866077929735184 + }, + "val": { + "mean_centered_corr": -0.02003737911581993, + "mean_pairwise_disagreement": NaN, + "member_std": 0.058194298297166824 + }, + "test": { + "mean_centered_corr": -0.01959315314888954, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07949266582727432 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 24, + 16, + 0, + 20, + 29 + ], + "head_selection_time": "0:00:00.087503" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.1-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.1-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.1-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.1-best-head-evaluation/0/report.json new file mode 100644 index 000000000..a34d94223 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.1-best-head-evaluation/0/report.json @@ -0,0 +1,643 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24287177140617242, + "mae": 0.12792739272117615, + "r2": 0.6231504678726196, + "score": -0.24287177140617242 + }, + "val": { + "rmse": 0.2388197411165918, + "mae": 0.13359415531158447, + "r2": 0.61902916431427, + "score": -0.2388197411165918 + }, + "test": { + "rmse": 0.2462523116085554, + "mae": 0.16663780808448792, + "r2": 0.5856186151504517, + "score": -0.2462523116085554 + } + }, + "time": "0:01:04.483422", + "diversity": { + "train": { + "mean_centered_corr": -0.018840299919247627, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04810040444135666 + }, + "val": { + "mean_centered_corr": -0.020352300256490707, + "mean_pairwise_disagreement": NaN, + "member_std": 0.061659909784793854 + }, + "test": { + "mean_centered_corr": -0.019725581631064415, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08170976489782333 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015081" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.1-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.1-evaluation/0.toml new file mode 100644 index 000000000..523756861 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.1-evaluation/0.toml @@ -0,0 +1,45 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.1-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.1-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.1-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.1-evaluation/0/report.json new file mode 100644 index 000000000..ad9df262e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.1-evaluation/0/report.json @@ -0,0 +1,639 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23279361488245573, + "mae": 0.12178773432970047, + "r2": 0.6537768840789795, + "score": -0.23279361488245573 + }, + "val": { + "rmse": 0.23523413233394516, + "mae": 0.12663741409778595, + "r2": 0.6303830146789551, + "score": -0.23523413233394516 + }, + "test": { + "rmse": 0.235149605552611, + "mae": 0.14915932714939117, + "r2": 0.6221424341201782, + "score": -0.235149605552611 + } + }, + "time": "0:01:04.483422", + "diversity": { + "train": { + "mean_centered_corr": -0.018840299919247627, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04810040444135666 + }, + "val": { + "mean_centered_corr": -0.020352300256490707, + "mean_pairwise_disagreement": NaN, + "member_std": 0.061659909784793854 + }, + "test": { + "mean_centered_corr": -0.019725581631064415, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08170976489782333 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.1-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.1-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.1-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.1-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..0a6b873ae --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.1-greedy-heads-evaluation/0/report.json @@ -0,0 +1,649 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23477394006764257, + "mae": 0.1248951405286789, + "r2": 0.6478613615036011, + "score": -0.23477394006764257 + }, + "val": { + "rmse": 0.23327114512588593, + "mae": 0.12512965500354767, + "r2": 0.6365260481834412, + "score": -0.23327114512588593 + }, + "test": { + "rmse": 0.23367141178994547, + "mae": 0.14663751423358917, + "r2": 0.6268780827522278, + "score": -0.23367141178994547 + } + }, + "time": "0:01:04.483422", + "diversity": { + "train": { + "mean_centered_corr": -0.018840299919247627, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04810040444135666 + }, + "val": { + "mean_centered_corr": -0.020352300256490707, + "mean_pairwise_disagreement": NaN, + "member_std": 0.061659909784793854 + }, + "test": { + "mean_centered_corr": -0.019725581631064415, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08170976489782333 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 0, + 21, + 25, + 19, + 0, + 28 + ], + "head_selection_time": "0:00:00.117459" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.2-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.2-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.2-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.2-best-head-evaluation/0/report.json new file mode 100644 index 000000000..3af7afec9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.2-best-head-evaluation/0/report.json @@ -0,0 +1,643 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.2, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.2, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.23290555777498698, + "mae": 0.12613320350646973, + "r2": 0.6534438729286194, + "score": -0.23290555777498698 + }, + "val": { + "rmse": 0.23866861156973368, + "mae": 0.13892269134521484, + "r2": 0.6195112466812134, + "score": -0.23866861156973368 + }, + "test": { + "rmse": 0.2497089599248454, + "mae": 0.17276842892169952, + "r2": 0.5739036202430725, + "score": -0.2497089599248454 + } + }, + "time": "0:01:08.920450", + "diversity": { + "train": { + "mean_centered_corr": -0.0169442780315876, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04900464415550232 + }, + "val": { + "mean_centered_corr": -0.02006734535098076, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06669005006551743 + }, + "test": { + "mean_centered_corr": -0.01514353696256876, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09919938445091248 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 22 + ], + "head_selection_time": "0:00:00.015245" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.2-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.2-evaluation/0.toml new file mode 100644 index 000000000..3912eb390 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.2-evaluation/0.toml @@ -0,0 +1,45 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.2 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.2-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.2-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.2-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.2-evaluation/0/report.json new file mode 100644 index 000000000..3e7f198e0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.2-evaluation/0/report.json @@ -0,0 +1,639 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.2, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.2, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2242219236860395, + "mae": 0.11819882690906525, + "r2": 0.6788040399551392, + "score": -0.2242219236860395 + }, + "val": { + "rmse": 0.23533942163580152, + "mae": 0.12796977162361145, + "r2": 0.6300520896911621, + "score": -0.23533942163580152 + }, + "test": { + "rmse": 0.24087782495581203, + "mae": 0.1581418365240097, + "r2": 0.6035090684890747, + "score": -0.24087782495581203 + } + }, + "time": "0:01:08.920450", + "diversity": { + "train": { + "mean_centered_corr": -0.0169442780315876, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04900464415550232 + }, + "val": { + "mean_centered_corr": -0.02006734535098076, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06669005006551743 + }, + "test": { + "mean_centered_corr": -0.01514353696256876, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09919938445091248 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.2-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.2-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.2-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.2-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..f2d7e30ac --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.2-greedy-heads-evaluation/0/report.json @@ -0,0 +1,654 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.2, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.2, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22441248349529827, + "mae": 0.1177876740694046, + "r2": 0.6782578825950623, + "score": -0.22441248349529827 + }, + "val": { + "rmse": 0.23337569983908155, + "mae": 0.12688370048999786, + "r2": 0.6362001895904541, + "score": -0.23337569983908155 + }, + "test": { + "rmse": 0.244512625524316, + "mae": 0.16398777067661285, + "r2": 0.5914528369903564, + "score": -0.244512625524316 + } + }, + "time": "0:01:08.920450", + "diversity": { + "train": { + "mean_centered_corr": -0.0169442780315876, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04900464415550232 + }, + "val": { + "mean_centered_corr": -0.02006734535098076, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06669005006551743 + }, + "test": { + "mean_centered_corr": -0.01514353696256876, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09919938445091248 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 22, + 19, + 25, + 21, + 29, + 18, + 8, + 1, + 19, + 25, + 12, + 4 + ], + "head_selection_time": "0:00:00.190618" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_cf_fisd-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_cf_fisd-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_cf_fisd-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_cf_fisd-best-head-evaluation/0/report.json new file mode 100644 index 000000000..fd0703522 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_cf_fisd-best-head-evaluation/0/report.json @@ -0,0 +1,653 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_cf_fisd-evaluation/_esam_diagnostics/esam_cf_fisd-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1406, + "metrics": { + "train": { + "rmse": 0.21463492388268743, + "mae": 0.11984407156705856, + "r2": 0.7056834697723389, + "score": -0.21463492388268743 + }, + "val": { + "rmse": 0.2404271956933659, + "mae": 0.14037233591079712, + "r2": 0.6138834357261658, + "score": -0.2404271956933659 + }, + "test": { + "rmse": 0.25751796428274565, + "mae": 0.17290249466896057, + "r2": 0.5468367338180542, + "score": -0.25751796428274565 + } + }, + "time": "0:02:42.897265", + "diversity": { + "train": { + "mean_centered_corr": -0.017316685989499092, + "mean_pairwise_disagreement": NaN, + "member_std": 0.050752341747283936 + }, + "val": { + "mean_centered_corr": -0.014526012353599072, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08017978072166443 + }, + "test": { + "mean_centered_corr": -0.008432705886662006, + "mean_pairwise_disagreement": NaN, + "member_std": 0.13299168646335602 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 27 + ], + "head_selection_time": "0:00:00.015175" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_cf_fisd-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_cf_fisd-evaluation/0.toml new file mode 100644 index 000000000..690d98000 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_cf_fisd-evaluation/0.toml @@ -0,0 +1,55 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_cf_fisd-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_cf_fisd-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_cf_fisd-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_cf_fisd-evaluation/0/report.json new file mode 100644 index 000000000..fc2c6d417 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_cf_fisd-evaluation/0/report.json @@ -0,0 +1,649 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_cf_fisd-evaluation/_esam_diagnostics/esam_cf_fisd-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1406, + "metrics": { + "train": { + "rmse": 0.20151074580440687, + "mae": 0.11064346134662628, + "r2": 0.740575909614563, + "score": -0.20151074580440687 + }, + "val": { + "rmse": 0.2345315410452097, + "mae": 0.12432665377855301, + "r2": 0.6325876712799072, + "score": -0.2345315410452097 + }, + "test": { + "rmse": 0.249503671176673, + "mae": 0.16559600830078125, + "r2": 0.5746039152145386, + "score": -0.249503671176673 + } + }, + "time": "0:02:42.897265", + "diversity": { + "train": { + "mean_centered_corr": -0.017316685989499092, + "mean_pairwise_disagreement": NaN, + "member_std": 0.050752341747283936 + }, + "val": { + "mean_centered_corr": -0.014526012353599072, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08017978072166443 + }, + "test": { + "mean_centered_corr": -0.008432705886662006, + "mean_pairwise_disagreement": NaN, + "member_std": 0.13299168646335602 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_cf_fisd-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_cf_fisd-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_cf_fisd-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_cf_fisd-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..ec09e51d6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_cf_fisd-greedy-heads-evaluation/0/report.json @@ -0,0 +1,662 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_cf_fisd-evaluation/_esam_diagnostics/esam_cf_fisd-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1406, + "metrics": { + "train": { + "rmse": 0.20170394200431338, + "mae": 0.10968871414661407, + "r2": 0.7400782108306885, + "score": -0.20170394200431338 + }, + "val": { + "rmse": 0.23329227219621285, + "mae": 0.12547366321086884, + "r2": 0.6364601850509644, + "score": -0.23329227219621285 + }, + "test": { + "rmse": 0.23856120001818734, + "mae": 0.15210896730422974, + "r2": 0.6110988855361938, + "score": -0.23856120001818734 + } + }, + "time": "0:02:42.897265", + "diversity": { + "train": { + "mean_centered_corr": -0.017316685989499092, + "mean_pairwise_disagreement": NaN, + "member_std": 0.050752341747283936 + }, + "val": { + "mean_centered_corr": -0.014526012353599072, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08017978072166443 + }, + "test": { + "mean_centered_corr": -0.008432705886662006, + "mean_pairwise_disagreement": NaN, + "member_std": 0.13299168646335602 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 27, + 4, + 21, + 22, + 7, + 31, + 19, + 23, + 24, + 26 + ], + "head_selection_time": "0:00:00.159301" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_mfb-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_mfb-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_mfb-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_mfb-best-head-evaluation/0/report.json new file mode 100644 index 000000000..61abe5749 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_mfb-best-head-evaluation/0/report.json @@ -0,0 +1,236 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_mfb-evaluation/_esam_diagnostics/esam_mfb-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.22732156992730132, + "mae": 0.12432415038347244, + "r2": 0.6698622703552246, + "score": -0.22732156992730132 + }, + "val": { + "rmse": 0.23962393015909736, + "mae": 0.13407635688781738, + "r2": 0.6164591312408447, + "score": -0.23962393015909736 + }, + "test": { + "rmse": 0.24513925399849143, + "mae": 0.16054488718509674, + "r2": 0.5893561840057373, + "score": -0.24513925399849143 + } + }, + "time": "0:01:44.775403", + "diversity": { + "train": { + "mean_centered_corr": -0.01788729801774025, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04960332438349724 + }, + "val": { + "mean_centered_corr": -0.01788644678890705, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07234996557235718 + }, + "test": { + "mean_centered_corr": -0.018943432718515396, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09170374274253845 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 12 + ], + "head_selection_time": "0:00:00.015426" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_mfb-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_mfb-evaluation/0.toml new file mode 100644 index 000000000..3060f47c8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_mfb-evaluation/0.toml @@ -0,0 +1,57 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_mfb-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_mfb-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_mfb-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_mfb-evaluation/0/report.json new file mode 100644 index 000000000..202e55306 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_mfb-evaluation/0/report.json @@ -0,0 +1,232 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_mfb-evaluation/_esam_diagnostics/esam_mfb-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.221803185432639, + "mae": 0.1178557351231575, + "r2": 0.6856963634490967, + "score": -0.221803185432639 + }, + "val": { + "rmse": 0.23493017564475285, + "mae": 0.12757235765457153, + "r2": 0.631337583065033, + "score": -0.23493017564475285 + }, + "test": { + "rmse": 0.23218164932047, + "mae": 0.14360341429710388, + "r2": 0.6316205263137817, + "score": -0.23218164932047 + } + }, + "time": "0:01:44.775403", + "diversity": { + "train": { + "mean_centered_corr": -0.01788729801774025, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04960332438349724 + }, + "val": { + "mean_centered_corr": -0.01788644678890705, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07234996557235718 + }, + "test": { + "mean_centered_corr": -0.018943432718515396, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09170374274253845 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_mfb-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_mfb-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_mfb-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_mfb-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..d5ae81ee8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_mfb-greedy-heads-evaluation/0/report.json @@ -0,0 +1,246 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_mfb-evaluation/_esam_diagnostics/esam_mfb-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.22279034307776457, + "mae": 0.11827730387449265, + "r2": 0.6828924417495728, + "score": -0.22279034307776457 + }, + "val": { + "rmse": 0.2336977870530739, + "mae": 0.12728972733020782, + "r2": 0.635195255279541, + "score": -0.2336977870530739 + }, + "test": { + "rmse": 0.23066386352840565, + "mae": 0.13997237384319305, + "r2": 0.6364210844039917, + "score": -0.23066386352840565 + } + }, + "time": "0:01:44.775403", + "diversity": { + "train": { + "mean_centered_corr": -0.01788729801774025, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04960332438349724 + }, + "val": { + "mean_centered_corr": -0.01788644678890705, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07234996557235718 + }, + "test": { + "mean_centered_corr": -0.018943432718515396, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09170374274253845 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 12, + 31, + 24, + 20, + 8, + 26, + 19, + 21, + 25, + 24, + 31 + ], + "head_selection_time": "0:00:00.175209" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..fc71d7ecb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.001-best-head-evaluation/0/report.json @@ -0,0 +1,200 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.001-evaluation/_esam_diagnostics/esam_rho0.001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2255253318194067, + "mae": 0.1253618746995926, + "r2": 0.6750589609146118, + "score": -0.2255253318194067 + }, + "val": { + "rmse": 0.23949031043262045, + "mae": 0.13341857492923737, + "r2": 0.6168867349624634, + "score": -0.23949031043262045 + }, + "test": { + "rmse": 0.24155888192007524, + "mae": 0.14752615988254547, + "r2": 0.6012638807296753, + "score": -0.24155888192007524 + } + }, + "time": "0:01:30.183340", + "diversity": { + "train": { + "mean_centered_corr": -0.017563171684741974, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048281844705343246 + }, + "val": { + "mean_centered_corr": -0.01754843257367611, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07056254893541336 + }, + "test": { + "mean_centered_corr": -0.016294721513986588, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09666775166988373 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19 + ], + "head_selection_time": "0:00:00.015040" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.001-evaluation/0.toml new file mode 100644 index 000000000..758520650 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.001-evaluation/0.toml @@ -0,0 +1,44 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.001 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.001-evaluation/0/report.json new file mode 100644 index 000000000..bdc1bdf6a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.001-evaluation/0/report.json @@ -0,0 +1,196 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.001-evaluation/_esam_diagnostics/esam_rho0.001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2197185345348766, + "mae": 0.11797432601451874, + "r2": 0.6915766000747681, + "score": -0.2197185345348766 + }, + "val": { + "rmse": 0.23467774706155475, + "mae": 0.1267256885766983, + "r2": 0.632129430770874, + "score": -0.23467774706155475 + }, + "test": { + "rmse": 0.23716614057842658, + "mae": 0.1508055180311203, + "r2": 0.6156339645385742, + "score": -0.23716614057842658 + } + }, + "time": "0:01:30.183340", + "diversity": { + "train": { + "mean_centered_corr": -0.017563171684741974, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048281844705343246 + }, + "val": { + "mean_centered_corr": -0.01754843257367611, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07056254893541336 + }, + "test": { + "mean_centered_corr": -0.016294721513986588, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09666775166988373 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..0855a0948 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,212 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.001-evaluation/_esam_diagnostics/esam_rho0.001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.22057000931073897, + "mae": 0.11852128803730011, + "r2": 0.6891815662384033, + "score": -0.22057000931073897 + }, + "val": { + "rmse": 0.23316146280351166, + "mae": 0.12596334517002106, + "r2": 0.6368677616119385, + "score": -0.23316146280351166 + }, + "test": { + "rmse": 0.23452524296935331, + "mae": 0.14668044447898865, + "r2": 0.6241463422775269, + "score": -0.23452524296935331 + } + }, + "time": "0:01:30.183340", + "diversity": { + "train": { + "mean_centered_corr": -0.017563171684741974, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048281844705343246 + }, + "val": { + "mean_centered_corr": -0.01754843257367611, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07056254893541336 + }, + "test": { + "mean_centered_corr": -0.016294721513986588, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09666775166988373 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19, + 4, + 16, + 12, + 0, + 25, + 31, + 3, + 22, + 29, + 12, + 27, + 20 + ], + "head_selection_time": "0:00:00.202696" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-best-head-evaluation/0/report.json new file mode 100644 index 000000000..bc82de281 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-best-head-evaluation/0/report.json @@ -0,0 +1,200 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-evaluation/_esam_diagnostics/esam_rho0.0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2261742520907439, + "mae": 0.12224072217941284, + "r2": 0.6731863021850586, + "score": -0.2261742520907439 + }, + "val": { + "rmse": 0.2377136850270048, + "mae": 0.13327811658382416, + "r2": 0.622549831867218, + "score": -0.2377136850270048 + }, + "test": { + "rmse": 0.262345673404661, + "mae": 0.17990553379058838, + "r2": 0.5296865105628967, + "score": -0.262345673404661 + } + }, + "time": "0:01:27.301753", + "diversity": { + "train": { + "mean_centered_corr": -0.01859835349023342, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04920431599020958 + }, + "val": { + "mean_centered_corr": -0.019710257649421692, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06811030209064484 + }, + "test": { + "mean_centered_corr": -0.013359437696635723, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09920918941497803 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015122" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-evaluation/0.toml new file mode 100644 index 000000000..05c49e679 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-evaluation/0.toml @@ -0,0 +1,44 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.0025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-evaluation/0/report.json new file mode 100644 index 000000000..5fee35a30 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-evaluation/0/report.json @@ -0,0 +1,196 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-evaluation/_esam_diagnostics/esam_rho0.0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22265827445732783, + "mae": 0.11788544058799744, + "r2": 0.6832683086395264, + "score": -0.22265827445732783 + }, + "val": { + "rmse": 0.23474623366768377, + "mae": 0.1270996779203415, + "r2": 0.6319146752357483, + "score": -0.23474623366768377 + }, + "test": { + "rmse": 0.23936563514587833, + "mae": 0.15535929799079895, + "r2": 0.6084716320037842, + "score": -0.23936563514587833 + } + }, + "time": "0:01:27.301753", + "diversity": { + "train": { + "mean_centered_corr": -0.01859835349023342, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04920431599020958 + }, + "val": { + "mean_centered_corr": -0.019710257649421692, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06811030209064484 + }, + "test": { + "mean_centered_corr": -0.013359437696635723, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09920918941497803 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..9ca33d310 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-greedy-heads-evaluation/0/report.json @@ -0,0 +1,207 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-evaluation/_esam_diagnostics/esam_rho0.0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22450348389711117, + "mae": 0.11922673135995865, + "r2": 0.6779968738555908, + "score": -0.22450348389711117 + }, + "val": { + "rmse": 0.23295802298295287, + "mae": 0.1258699893951416, + "r2": 0.6375011801719666, + "score": -0.23295802298295287 + }, + "test": { + "rmse": 0.23903366020873654, + "mae": 0.15511320531368256, + "r2": 0.6095569133758545, + "score": -0.23903366020873654 + } + }, + "time": "0:01:27.301753", + "diversity": { + "train": { + "mean_centered_corr": -0.01859835349023342, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04920431599020958 + }, + "val": { + "mean_centered_corr": -0.019710257649421692, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06811030209064484 + }, + "test": { + "mean_centered_corr": -0.013359437696635723, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09920918941497803 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 22, + 12, + 16, + 21, + 19, + 25, + 29 + ], + "head_selection_time": "0:00:00.128997" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..5ac0f4a22 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.005-best-head-evaluation/0/report.json @@ -0,0 +1,200 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.005-evaluation/_esam_diagnostics/esam_rho0.005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2276013825171354, + "mae": 0.12862664461135864, + "r2": 0.6690490245819092, + "score": -0.2276013825171354 + }, + "val": { + "rmse": 0.2398028724080995, + "mae": 0.1366567611694336, + "r2": 0.6158860921859741, + "score": -0.2398028724080995 + }, + "test": { + "rmse": 0.23711371881317633, + "mae": 0.14537189900875092, + "r2": 0.6158038973808289, + "score": -0.23711371881317633 + } + }, + "time": "0:01:29.953970", + "diversity": { + "train": { + "mean_centered_corr": -0.017507946118712425, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0488579161465168 + }, + "val": { + "mean_centered_corr": -0.017691537737846375, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0768243595957756 + }, + "test": { + "mean_centered_corr": -0.014683594927191734, + "mean_pairwise_disagreement": NaN, + "member_std": 0.10134773701429367 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19 + ], + "head_selection_time": "0:00:00.015185" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.005-evaluation/0.toml new file mode 100644 index 000000000..f5a2c1fd1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.005-evaluation/0.toml @@ -0,0 +1,44 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.005-evaluation/0/report.json new file mode 100644 index 000000000..393470cd9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.005-evaluation/0/report.json @@ -0,0 +1,196 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.005-evaluation/_esam_diagnostics/esam_rho0.005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.21979515700408836, + "mae": 0.12170511484146118, + "r2": 0.6913614869117737, + "score": -0.21979515700408836 + }, + "val": { + "rmse": 0.23533073111501152, + "mae": 0.12932802736759186, + "r2": 0.6300793886184692, + "score": -0.23533073111501152 + }, + "test": { + "rmse": 0.24162357572341994, + "mae": 0.15881586074829102, + "r2": 0.6010502576828003, + "score": -0.24162357572341994 + } + }, + "time": "0:01:29.953970", + "diversity": { + "train": { + "mean_centered_corr": -0.017507946118712425, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0488579161465168 + }, + "val": { + "mean_centered_corr": -0.017691537737846375, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0768243595957756 + }, + "test": { + "mean_centered_corr": -0.014683594927191734, + "mean_pairwise_disagreement": NaN, + "member_std": 0.10134773701429367 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..6fcc28b95 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,209 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.005-evaluation/_esam_diagnostics/esam_rho0.005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.22114994595231807, + "mae": 0.12296636402606964, + "r2": 0.6875449419021606, + "score": -0.22114994595231807 + }, + "val": { + "rmse": 0.23380275644783471, + "mae": 0.12705549597740173, + "r2": 0.6348674893379211, + "score": -0.23380275644783471 + }, + "test": { + "rmse": 0.24257603366460162, + "mae": 0.1593162715435028, + "r2": 0.5978988409042358, + "score": -0.24257603366460162 + } + }, + "time": "0:01:29.953970", + "diversity": { + "train": { + "mean_centered_corr": -0.017507946118712425, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0488579161465168 + }, + "val": { + "mean_centered_corr": -0.017691537737846375, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0768243595957756 + }, + "test": { + "mean_centered_corr": -0.014683594927191734, + "mean_pairwise_disagreement": NaN, + "member_std": 0.10134773701429367 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19, + 25, + 0, + 16, + 30, + 22, + 25, + 31, + 16, + 22 + ], + "head_selection_time": "0:00:00.158575" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..c9c43f347 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.01-best-head-evaluation/0/report.json @@ -0,0 +1,200 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.01-evaluation/_esam_diagnostics/esam_rho0.01-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2265011030074498, + "mae": 0.12218555063009262, + "r2": 0.6722410917282104, + "score": -0.2265011030074498 + }, + "val": { + "rmse": 0.23855772550880586, + "mae": 0.1326008141040802, + "r2": 0.6198647022247314, + "score": -0.23855772550880586 + }, + "test": { + "rmse": 0.24435949051951408, + "mae": 0.1609477698802948, + "r2": 0.5919644832611084, + "score": -0.24435949051951408 + } + }, + "time": "0:01:26.741342", + "diversity": { + "train": { + "mean_centered_corr": -0.018773315474390984, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047917984426021576 + }, + "val": { + "mean_centered_corr": -0.01945357583463192, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06856771558523178 + }, + "test": { + "mean_centered_corr": -0.015788640826940536, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09582500904798508 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 18 + ], + "head_selection_time": "0:00:00.015262" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.01-evaluation/0.toml new file mode 100644 index 000000000..1f58fca60 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.01-evaluation/0.toml @@ -0,0 +1,44 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.01 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.01-evaluation/0/report.json new file mode 100644 index 000000000..95ee3f701 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.01-evaluation/0/report.json @@ -0,0 +1,196 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.01-evaluation/_esam_diagnostics/esam_rho0.01-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22394801918202625, + "mae": 0.12045823782682419, + "r2": 0.6795883178710938, + "score": -0.22394801918202625 + }, + "val": { + "rmse": 0.2345959019738195, + "mae": 0.12734748423099518, + "r2": 0.6323859691619873, + "score": -0.2345959019738195 + }, + "test": { + "rmse": 0.23852799100834732, + "mae": 0.15464624762535095, + "r2": 0.611207127571106, + "score": -0.23852799100834732 + } + }, + "time": "0:01:26.741342", + "diversity": { + "train": { + "mean_centered_corr": -0.018773315474390984, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047917984426021576 + }, + "val": { + "mean_centered_corr": -0.01945357583463192, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06856771558523178 + }, + "test": { + "mean_centered_corr": -0.015788640826940536, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09582500904798508 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..51c3bd29e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,206 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.01-evaluation/_esam_diagnostics/esam_rho0.01-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2237227812786278, + "mae": 0.12099222093820572, + "r2": 0.6802325248718262, + "score": -0.2237227812786278 + }, + "val": { + "rmse": 0.23321113093450146, + "mae": 0.12660610675811768, + "r2": 0.6367130279541016, + "score": -0.23321113093450146 + }, + "test": { + "rmse": 0.23887208479744804, + "mae": 0.15645892918109894, + "r2": 0.610084593296051, + "score": -0.23887208479744804 + } + }, + "time": "0:01:26.741342", + "diversity": { + "train": { + "mean_centered_corr": -0.018773315474390984, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047917984426021576 + }, + "val": { + "mean_centered_corr": -0.01945357583463192, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06856771558523178 + }, + "test": { + "mean_centered_corr": -0.015788640826940536, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09582500904798508 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 18, + 4, + 19, + 1, + 20, + 21, + 25 + ], + "head_selection_time": "0:00:00.115142" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_cf_fisd-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_cf_fisd-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_cf_fisd-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_cf_fisd-best-head-evaluation/0/report.json new file mode 100644 index 000000000..27e477e2c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_cf_fisd-best-head-evaluation/0/report.json @@ -0,0 +1,679 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.23359725177653518, + "mae": 0.12286694347858429, + "r2": 0.651382327079773, + "score": -0.23359725177653518 + }, + "val": { + "rmse": 0.23765498863853576, + "mae": 0.13634204864501953, + "r2": 0.6227362155914307, + "score": -0.23765498863853576 + }, + "test": { + "rmse": 0.2536206620016002, + "mae": 0.175298810005188, + "r2": 0.5604493618011475, + "score": -0.2536206620016002 + } + }, + "time": "0:01:22.331704", + "diversity": { + "train": { + "mean_centered_corr": -0.018672151491045952, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0502934493124485 + }, + "val": { + "mean_centered_corr": -0.01943311281502247, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06968922913074493 + }, + "test": { + "mean_centered_corr": -0.020102957263588905, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09299962222576141 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25 + ], + "head_selection_time": "0:00:00.015321" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_cf_fisd-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_cf_fisd-evaluation/0.toml new file mode 100644 index 000000000..e9f43e127 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_cf_fisd-evaluation/0.toml @@ -0,0 +1,58 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_cf_fisd-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_cf_fisd-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_cf_fisd-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_cf_fisd-evaluation/0/report.json new file mode 100644 index 000000000..78c9ce2d6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_cf_fisd-evaluation/0/report.json @@ -0,0 +1,675 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.22208876822485193, + "mae": 0.11730500310659409, + "r2": 0.6848864555358887, + "score": -0.22208876822485193 + }, + "val": { + "rmse": 0.23508488120210985, + "mae": 0.12783657014369965, + "r2": 0.6308518648147583, + "score": -0.23508488120210985 + }, + "test": { + "rmse": 0.23568789402876444, + "mae": 0.14989204704761505, + "r2": 0.6204105615615845, + "score": -0.23568789402876444 + } + }, + "time": "0:01:22.331704", + "diversity": { + "train": { + "mean_centered_corr": -0.018672151491045952, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0502934493124485 + }, + "val": { + "mean_centered_corr": -0.01943311281502247, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06968922913074493 + }, + "test": { + "mean_centered_corr": -0.020102957263588905, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09299962222576141 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_cf_fisd-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_cf_fisd-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_cf_fisd-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_cf_fisd-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..635150edd --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_cf_fisd-greedy-heads-evaluation/0/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.22343022528056483, + "mae": 0.11760932952165604, + "r2": 0.6810683012008667, + "score": -0.22343022528056483 + }, + "val": { + "rmse": 0.23320900639668657, + "mae": 0.12497496604919434, + "r2": 0.6367197036743164, + "score": -0.23320900639668657 + }, + "test": { + "rmse": 0.23362227205150313, + "mae": 0.14383481442928314, + "r2": 0.6270350217819214, + "score": -0.23362227205150313 + } + }, + "time": "0:01:22.331704", + "diversity": { + "train": { + "mean_centered_corr": -0.018672151491045952, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0502934493124485 + }, + "val": { + "mean_centered_corr": -0.01943311281502247, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06968922913074493 + }, + "test": { + "mean_centered_corr": -0.020102957263588905, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09299962222576141 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25, + 20, + 29, + 19, + 0, + 12, + 4, + 20, + 21, + 19 + ], + "head_selection_time": "0:00:00.159678" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.7-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.7-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.7-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.7-best-head-evaluation/0/report.json new file mode 100644 index 000000000..adca48472 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.7-best-head-evaluation/0/report.json @@ -0,0 +1,226 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.7039222121238708, + "min_features_per_member": 253.0, + "mean_features_per_member": 275.9375, + "max_features_per_member": 294.0, + "min_members_per_feature": 15.0, + "mean_members_per_feature": 22.525510787963867, + "max_members_per_feature": 30.0, + "mean_pairwise_jaccard": 0.542541533357324, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.23190653592940091, + "mae": 0.12970374524593353, + "r2": 0.6564105153083801, + "score": -0.23190653592940091 + }, + "val": { + "rmse": 0.24062365494389312, + "mae": 0.13845081627368927, + "r2": 0.6132521629333496, + "score": -0.24062365494389312 + }, + "test": { + "rmse": 0.24557535127152327, + "mae": 0.16125304996967316, + "r2": 0.5878938436508179, + "score": -0.24557535127152327 + } + }, + "time": "0:00:59.832262", + "diversity": { + "train": { + "mean_centered_corr": -0.018218370154500008, + "mean_pairwise_disagreement": NaN, + "member_std": 0.051916833966970444 + }, + "val": { + "mean_centered_corr": -0.019246703013777733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07262936979532242 + }, + "test": { + "mean_centered_corr": -0.017951499670743942, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09685395658016205 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0 + ], + "head_selection_time": "0:00:00.015144" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.7-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.7-evaluation/0.toml new file mode 100644 index 000000000..7fa20abf9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.7-evaluation/0.toml @@ -0,0 +1,47 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.7 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.7-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.7-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.7-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.7-evaluation/0/report.json new file mode 100644 index 000000000..e3eaa700c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.7-evaluation/0/report.json @@ -0,0 +1,222 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.7039222121238708, + "min_features_per_member": 253.0, + "mean_features_per_member": 275.9375, + "max_features_per_member": 294.0, + "min_members_per_feature": 15.0, + "mean_members_per_feature": 22.525510787963867, + "max_members_per_feature": 30.0, + "mean_pairwise_jaccard": 0.542541533357324, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.22535369940802735, + "mae": 0.11960747092962265, + "r2": 0.6755533218383789, + "score": -0.22535369940802735 + }, + "val": { + "rmse": 0.23535581244592554, + "mae": 0.12930907309055328, + "r2": 0.6300005316734314, + "score": -0.23535581244592554 + }, + "test": { + "rmse": 0.23562693018337708, + "mae": 0.14956335723400116, + "r2": 0.6206068992614746, + "score": -0.23562693018337708 + } + }, + "time": "0:00:59.832262", + "diversity": { + "train": { + "mean_centered_corr": -0.018218370154500008, + "mean_pairwise_disagreement": NaN, + "member_std": 0.051916833966970444 + }, + "val": { + "mean_centered_corr": -0.019246703013777733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07262936979532242 + }, + "test": { + "mean_centered_corr": -0.017951499670743942, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09685395658016205 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.7-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.7-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.7-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.7-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..3bb540523 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.7-greedy-heads-evaluation/0/report.json @@ -0,0 +1,236 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.7039222121238708, + "min_features_per_member": 253.0, + "mean_features_per_member": 275.9375, + "max_features_per_member": 294.0, + "min_members_per_feature": 15.0, + "mean_members_per_feature": 22.525510787963867, + "max_members_per_feature": 30.0, + "mean_pairwise_jaccard": 0.542541533357324, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.22381426184098463, + "mae": 0.11932115256786346, + "r2": 0.6799709796905518, + "score": -0.22381426184098463 + }, + "val": { + "rmse": 0.23323099362803706, + "mae": 0.12578125298023224, + "r2": 0.6366511583328247, + "score": -0.23323099362803706 + }, + "test": { + "rmse": 0.2357129689640159, + "mae": 0.149356409907341, + "r2": 0.620329737663269, + "score": -0.2357129689640159 + } + }, + "time": "0:00:59.832262", + "diversity": { + "train": { + "mean_centered_corr": -0.018218370154500008, + "mean_pairwise_disagreement": NaN, + "member_std": 0.051916833966970444 + }, + "val": { + "mean_centered_corr": -0.019246703013777733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07262936979532242 + }, + "test": { + "mean_centered_corr": -0.017951499670743942, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09685395658016205 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0, + 2, + 28, + 20, + 1, + 19, + 21, + 14, + 31, + 20, + 12 + ], + "head_selection_time": "0:00:00.173195" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.8-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.8-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.8-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.8-best-head-evaluation/0/report.json new file mode 100644 index 000000000..688b587ad --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.8-best-head-evaluation/0/report.json @@ -0,0 +1,226 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.8038105964660645, + "min_features_per_member": 298.0, + "mean_features_per_member": 315.09375, + "max_features_per_member": 335.0, + "min_members_per_feature": 19.0, + "mean_members_per_feature": 25.721939086914062, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.6715097166219348, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2316786318343145, + "mae": 0.12621846795082092, + "r2": 0.6570855379104614, + "score": -0.2316786318343145 + }, + "val": { + "rmse": 0.2380346366558266, + "mae": 0.1380215585231781, + "r2": 0.6215299367904663, + "score": -0.2380346366558266 + }, + "test": { + "rmse": 0.2579307429494123, + "mae": 0.17571206390857697, + "r2": 0.5453828573226929, + "score": -0.2579307429494123 + } + }, + "time": "0:00:59.854964", + "diversity": { + "train": { + "mean_centered_corr": -0.017583154141902924, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05003640055656433 + }, + "val": { + "mean_centered_corr": -0.018760371953248978, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07571490854024887 + }, + "test": { + "mean_centered_corr": -0.019624220207333565, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09894119203090668 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25 + ], + "head_selection_time": "0:00:00.016145" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.8-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.8-evaluation/0.toml new file mode 100644 index 000000000..d44bc73bf --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.8-evaluation/0.toml @@ -0,0 +1,47 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.8 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.8-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.8-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.8-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.8-evaluation/0/report.json new file mode 100644 index 000000000..66077afb4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.8-evaluation/0/report.json @@ -0,0 +1,222 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.8038105964660645, + "min_features_per_member": 298.0, + "mean_features_per_member": 315.09375, + "max_features_per_member": 335.0, + "min_members_per_feature": 19.0, + "mean_members_per_feature": 25.721939086914062, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.6715097166219348, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.22081238228152064, + "mae": 0.11986075341701508, + "r2": 0.6884980797767639, + "score": -0.22081238228152064 + }, + "val": { + "rmse": 0.2341279953511937, + "mae": 0.12804140150547028, + "r2": 0.6338509321212769, + "score": -0.2341279953511937 + }, + "test": { + "rmse": 0.23402994505001048, + "mae": 0.14647243916988373, + "r2": 0.6257321834564209, + "score": -0.23402994505001048 + } + }, + "time": "0:00:59.854964", + "diversity": { + "train": { + "mean_centered_corr": -0.017583154141902924, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05003640055656433 + }, + "val": { + "mean_centered_corr": -0.018760371953248978, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07571490854024887 + }, + "test": { + "mean_centered_corr": -0.019624220207333565, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09894119203090668 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.8-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.8-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.8-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.8-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..b613a292e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.8-greedy-heads-evaluation/0/report.json @@ -0,0 +1,235 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.8038105964660645, + "min_features_per_member": 298.0, + "mean_features_per_member": 315.09375, + "max_features_per_member": 335.0, + "min_members_per_feature": 19.0, + "mean_members_per_feature": 25.721939086914062, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.6715097166219348, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.22132344126163775, + "mae": 0.11854095757007599, + "r2": 0.6870545148849487, + "score": -0.22132344126163775 + }, + "val": { + "rmse": 0.23237596587359466, + "mae": 0.1258111149072647, + "r2": 0.639310359954834, + "score": -0.23237596587359466 + }, + "test": { + "rmse": 0.23064696162155762, + "mae": 0.1411736011505127, + "r2": 0.6364743709564209, + "score": -0.23064696162155762 + } + }, + "time": "0:00:59.854964", + "diversity": { + "train": { + "mean_centered_corr": -0.017583154141902924, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05003640055656433 + }, + "val": { + "mean_centered_corr": -0.018760371953248978, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07571490854024887 + }, + "test": { + "mean_centered_corr": -0.019624220207333565, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09894119203090668 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25, + 28, + 2, + 27, + 12, + 24, + 9, + 3, + 21, + 28 + ], + "head_selection_time": "0:00:00.166841" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.9-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.9-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.9-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.9-best-head-evaluation/0/report.json new file mode 100644 index 000000000..ed475ec53 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.9-best-head-evaluation/0/report.json @@ -0,0 +1,226 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1184, + "metrics": { + "train": { + "rmse": 0.2105902710250961, + "mae": 0.11857529729604721, + "r2": 0.716671347618103, + "score": -0.2105902710250961 + }, + "val": { + "rmse": 0.24356476580119216, + "mae": 0.1343512386083603, + "r2": 0.6037400364875793, + "score": -0.24356476580119216 + }, + "test": { + "rmse": 0.2459548830871388, + "mae": 0.14966677129268646, + "r2": 0.5866190195083618, + "score": -0.2459548830871388 + } + }, + "time": "0:01:08.783577", + "diversity": { + "train": { + "mean_centered_corr": -0.016982607543468475, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05261818319559097 + }, + "val": { + "mean_centered_corr": -0.017025893554091454, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08680207282304764 + }, + "test": { + "mean_centered_corr": -0.01696028746664524, + "mean_pairwise_disagreement": NaN, + "member_std": 0.11552654951810837 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31 + ], + "head_selection_time": "0:00:00.015090" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.9-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.9-evaluation/0.toml new file mode 100644 index 000000000..da979e186 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.9-evaluation/0.toml @@ -0,0 +1,47 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.9-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.9-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.9-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.9-evaluation/0/report.json new file mode 100644 index 000000000..61c8effcd --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.9-evaluation/0/report.json @@ -0,0 +1,222 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1184, + "metrics": { + "train": { + "rmse": 0.20459342170445438, + "mae": 0.11115512996912003, + "r2": 0.732577919960022, + "score": -0.20459342170445438 + }, + "val": { + "rmse": 0.23520949733542967, + "mae": 0.1269780397415161, + "r2": 0.6304603815078735, + "score": -0.23520949733542967 + }, + "test": { + "rmse": 0.23880983570262945, + "mae": 0.15338461101055145, + "r2": 0.6102877855300903, + "score": -0.23880983570262945 + } + }, + "time": "0:01:08.783577", + "diversity": { + "train": { + "mean_centered_corr": -0.016982607543468475, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05261818319559097 + }, + "val": { + "mean_centered_corr": -0.017025893554091454, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08680207282304764 + }, + "test": { + "mean_centered_corr": -0.01696028746664524, + "mean_pairwise_disagreement": NaN, + "member_std": 0.11552654951810837 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.9-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.9-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.9-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.9-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..8321a1cc8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.9-greedy-heads-evaluation/0/report.json @@ -0,0 +1,234 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1184, + "metrics": { + "train": { + "rmse": 0.20332438513408257, + "mae": 0.11058838665485382, + "r2": 0.7358851432800293, + "score": -0.20332438513408257 + }, + "val": { + "rmse": 0.233840053737128, + "mae": 0.12618696689605713, + "r2": 0.6347509622573853, + "score": -0.233840053737128 + }, + "test": { + "rmse": 0.23548819505307594, + "mae": 0.14876849949359894, + "r2": 0.6210535764694214, + "score": -0.23548819505307594 + } + }, + "time": "0:01:08.783577", + "diversity": { + "train": { + "mean_centered_corr": -0.016982607543468475, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05261818319559097 + }, + "val": { + "mean_centered_corr": -0.017025893554091454, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08680207282304764 + }, + "test": { + "mean_centered_corr": -0.01696028746664524, + "mean_pairwise_disagreement": NaN, + "member_std": 0.11552654951810837 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31, + 27, + 25, + 19, + 8, + 4, + 28, + 17, + 15 + ], + "head_selection_time": "0:00:00.144349" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.95-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.95-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.95-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.95-best-head-evaluation/0/report.json new file mode 100644 index 000000000..b8b7f9147 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.95-best-head-evaluation/0/report.json @@ -0,0 +1,226 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.22956111765569162, + "mae": 0.12580275535583496, + "r2": 0.6633252501487732, + "score": -0.22956111765569162 + }, + "val": { + "rmse": 0.23835928999137548, + "mae": 0.13410992920398712, + "r2": 0.6204968094825745, + "score": -0.23835928999137548 + }, + "test": { + "rmse": 0.27696045729988356, + "mae": 0.19395296275615692, + "r2": 0.4758263826370239, + "score": -0.27696045729988356 + } + }, + "time": "0:01:00.082091", + "diversity": { + "train": { + "mean_centered_corr": -0.017180385068058968, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04938776046037674 + }, + "val": { + "mean_centered_corr": -0.018672365695238113, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06963518261909485 + }, + "test": { + "mean_centered_corr": -0.01587279699742794, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09464479982852936 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25 + ], + "head_selection_time": "0:00:00.015230" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.95-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.95-evaluation/0.toml new file mode 100644 index 000000000..035cc781b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.95-evaluation/0.toml @@ -0,0 +1,47 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.95 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.95-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.95-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.95-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.95-evaluation/0/report.json new file mode 100644 index 000000000..a9a095271 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.95-evaluation/0/report.json @@ -0,0 +1,222 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.21949891059220633, + "mae": 0.11622610688209534, + "r2": 0.6921929121017456, + "score": -0.21949891059220633 + }, + "val": { + "rmse": 0.2352128391676174, + "mae": 0.12513305246829987, + "r2": 0.6304498910903931, + "score": -0.2352128391676174 + }, + "test": { + "rmse": 0.23206813724553327, + "mae": 0.14319945871829987, + "r2": 0.6319806575775146, + "score": -0.23206813724553327 + } + }, + "time": "0:01:00.082091", + "diversity": { + "train": { + "mean_centered_corr": -0.017180385068058968, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04938776046037674 + }, + "val": { + "mean_centered_corr": -0.018672365695238113, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06963518261909485 + }, + "test": { + "mean_centered_corr": -0.01587279699742794, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09464479982852936 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.95-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.95-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.95-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.95-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..f5a5ce75c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.95-greedy-heads-evaluation/0/report.json @@ -0,0 +1,235 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.21898836534384564, + "mae": 0.1169113889336586, + "r2": 0.6936231255531311, + "score": -0.21898836534384564 + }, + "val": { + "rmse": 0.23254505067097037, + "mae": 0.12637269496917725, + "r2": 0.6387852430343628, + "score": -0.23254505067097037 + }, + "test": { + "rmse": 0.23506758402869707, + "mae": 0.1480458676815033, + "r2": 0.622406005859375, + "score": -0.23506758402869707 + } + }, + "time": "0:01:00.082091", + "diversity": { + "train": { + "mean_centered_corr": -0.017180385068058968, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04938776046037674 + }, + "val": { + "mean_centered_corr": -0.018672365695238113, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06963518261909485 + }, + "test": { + "mean_centered_corr": -0.01587279699742794, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09464479982852936 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25, + 4, + 12, + 19, + 0, + 21, + 4, + 20, + 31, + 0 + ], + "head_selection_time": "0:00:00.161207" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_cf_fisd-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_cf_fisd-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_cf_fisd-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_cf_fisd-best-head-evaluation/0/report.json new file mode 100644 index 000000000..c1b4cfc5c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_cf_fisd-best-head-evaluation/0/report.json @@ -0,0 +1,656 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.21924836575226364, + "mae": 0.12353650480508804, + "r2": 0.6928951740264893, + "score": -0.21924836575226364 + }, + "val": { + "rmse": 0.23788056532412807, + "mae": 0.13566423952579498, + "r2": 0.6220196485519409, + "score": -0.23788056532412807 + }, + "test": { + "rmse": 0.24833117735063384, + "mae": 0.1667243391275406, + "r2": 0.5785926580429077, + "score": -0.24833117735063384 + } + }, + "time": "0:02:05.644832", + "diversity": { + "train": { + "mean_centered_corr": -0.017985021695494652, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047170910984277725 + }, + "val": { + "mean_centered_corr": -0.017544301226735115, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06983517855405807 + }, + "test": { + "mean_centered_corr": -0.015103532001376152, + "mean_pairwise_disagreement": NaN, + "member_std": 0.10299047082662582 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21 + ], + "head_selection_time": "0:00:00.015293" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_cf_fisd-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_cf_fisd-evaluation/0.toml new file mode 100644 index 000000000..d46520305 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_cf_fisd-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_cf_fisd-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_cf_fisd-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_cf_fisd-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_cf_fisd-evaluation/0/report.json new file mode 100644 index 000000000..58438d978 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_cf_fisd-evaluation/0/report.json @@ -0,0 +1,652 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.21744429514871078, + "mae": 0.11533833295106888, + "r2": 0.6979283690452576, + "score": -0.21744429514871078 + }, + "val": { + "rmse": 0.23507577716825093, + "mae": 0.12675879895687103, + "r2": 0.6308804750442505, + "score": -0.23507577716825093 + }, + "test": { + "rmse": 0.2401363436403948, + "mae": 0.15580184757709503, + "r2": 0.6059463024139404, + "score": -0.2401363436403948 + } + }, + "time": "0:02:05.644832", + "diversity": { + "train": { + "mean_centered_corr": -0.017985021695494652, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047170910984277725 + }, + "val": { + "mean_centered_corr": -0.017544301226735115, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06983517855405807 + }, + "test": { + "mean_centered_corr": -0.015103532001376152, + "mean_pairwise_disagreement": NaN, + "member_std": 0.10299047082662582 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_cf_fisd-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_cf_fisd-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_cf_fisd-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_cf_fisd-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..be77993ab --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_cf_fisd-greedy-heads-evaluation/0/report.json @@ -0,0 +1,668 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.21849668686011264, + "mae": 0.11518727242946625, + "r2": 0.6949973702430725, + "score": -0.21849668686011264 + }, + "val": { + "rmse": 0.233603375533187, + "mae": 0.12658587098121643, + "r2": 0.6354899406433105, + "score": -0.233603375533187 + }, + "test": { + "rmse": 0.23596160620261294, + "mae": 0.14986030757427216, + "r2": 0.6195283532142639, + "score": -0.23596160620261294 + } + }, + "time": "0:02:05.644832", + "diversity": { + "train": { + "mean_centered_corr": -0.017985021695494652, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047170910984277725 + }, + "val": { + "mean_centered_corr": -0.017544301226735115, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06983517855405807 + }, + "test": { + "mean_centered_corr": -0.015103532001376152, + "mean_pairwise_disagreement": NaN, + "member_std": 0.10299047082662582 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21, + 4, + 26, + 9, + 1, + 6, + 22, + 4, + 19, + 0, + 26, + 20, + 29 + ], + "head_selection_time": "0:00:00.204485" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_esam-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_esam-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_esam-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_esam-best-head-evaluation/0/report.json new file mode 100644 index 000000000..8e988752a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_esam-best-head-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_esam-evaluation/_esam_diagnostics/rla_esam-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2353728431166263, + "mae": 0.12022902816534042, + "r2": 0.6460624933242798, + "score": -0.2353728431166263 + }, + "val": { + "rmse": 0.2398484083586863, + "mae": 0.13140499591827393, + "r2": 0.615740180015564, + "score": -0.2398484083586863 + }, + "test": { + "rmse": 0.2374880282622196, + "mae": 0.15091650187969208, + "r2": 0.6145899295806885, + "score": -0.2374880282622196 + } + }, + "time": "0:02:41.758491", + "diversity": { + "train": { + "mean_centered_corr": -0.01889662630856037, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04478425905108452 + }, + "val": { + "mean_centered_corr": -0.02039206400513649, + "mean_pairwise_disagreement": NaN, + "member_std": 0.063406802713871 + }, + "test": { + "mean_centered_corr": -0.01742168515920639, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0834045484662056 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 28 + ], + "head_selection_time": "0:00:00.015316" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_esam-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_esam-evaluation/0.toml new file mode 100644 index 000000000..d960be7a0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_esam-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_esam-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_esam-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_esam-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_esam-evaluation/0/report.json new file mode 100644 index 000000000..4c0caaef2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_esam-evaluation/0/report.json @@ -0,0 +1,209 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_esam-evaluation/_esam_diagnostics/rla_esam-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23095795064666533, + "mae": 0.12093835324048996, + "r2": 0.6592155694961548, + "score": -0.23095795064666533 + }, + "val": { + "rmse": 0.23528677495791916, + "mae": 0.1253853291273117, + "r2": 0.6302175521850586, + "score": -0.23528677495791916 + }, + "test": { + "rmse": 0.23252163677391782, + "mae": 0.1432647854089737, + "r2": 0.6305409073829651, + "score": -0.23252163677391782 + } + }, + "time": "0:02:41.758491", + "diversity": { + "train": { + "mean_centered_corr": -0.01889662630856037, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04478425905108452 + }, + "val": { + "mean_centered_corr": -0.02039206400513649, + "mean_pairwise_disagreement": NaN, + "member_std": 0.063406802713871 + }, + "test": { + "mean_centered_corr": -0.01742168515920639, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0834045484662056 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_esam-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_esam-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_esam-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_esam-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..7815e3439 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_esam-greedy-heads-evaluation/0/report.json @@ -0,0 +1,217 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_esam-evaluation/_esam_diagnostics/rla_esam-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23231665048542627, + "mae": 0.12011752277612686, + "r2": 0.6551941633224487, + "score": -0.23231665048542627 + }, + "val": { + "rmse": 0.2336232925794531, + "mae": 0.12742893397808075, + "r2": 0.6354278326034546, + "score": -0.2336232925794531 + }, + "test": { + "rmse": 0.233028022164673, + "mae": 0.1476990431547165, + "r2": 0.6289299726486206, + "score": -0.233028022164673 + } + }, + "time": "0:02:41.758491", + "diversity": { + "train": { + "mean_centered_corr": -0.01889662630856037, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04478425905108452 + }, + "val": { + "mean_centered_corr": -0.02039206400513649, + "mean_pairwise_disagreement": NaN, + "member_std": 0.063406802713871 + }, + "test": { + "mean_centered_corr": -0.01742168515920639, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0834045484662056 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 28, + 16, + 12, + 4, + 18 + ], + "head_selection_time": "0:00:00.087428" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_mfb-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_mfb-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_mfb-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_mfb-best-head-evaluation/0/report.json new file mode 100644 index 000000000..f165e1fc0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_mfb-best-head-evaluation/0/report.json @@ -0,0 +1,239 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.22945006775369764, + "mae": 0.12472175061702728, + "r2": 0.6636508703231812, + "score": -0.22945006775369764 + }, + "val": { + "rmse": 0.2399809438744672, + "mae": 0.13324783742427826, + "r2": 0.6153154373168945, + "score": -0.2399809438744672 + }, + "test": { + "rmse": 0.24194702392020134, + "mae": 0.15345057845115662, + "r2": 0.599981427192688, + "score": -0.24194702392020134 + } + }, + "time": "0:01:56.838272", + "diversity": { + "train": { + "mean_centered_corr": -0.018120070919394493, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04867309704422951 + }, + "val": { + "mean_centered_corr": -0.01935560256242752, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07195134460926056 + }, + "test": { + "mean_centered_corr": -0.019733797758817673, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09048754721879959 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25 + ], + "head_selection_time": "0:00:00.015258" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_mfb-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_mfb-evaluation/0.toml new file mode 100644 index 000000000..25b3f5ef5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_mfb-evaluation/0.toml @@ -0,0 +1,54 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_mfb-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_mfb-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_mfb-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_mfb-evaluation/0/report.json new file mode 100644 index 000000000..59ddd06ea --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_mfb-evaluation/0/report.json @@ -0,0 +1,235 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2171101669154321, + "mae": 0.11589445918798447, + "r2": 0.698855996131897, + "score": -0.2171101669154321 + }, + "val": { + "rmse": 0.23551030167646542, + "mae": 0.12883572280406952, + "r2": 0.6295146346092224, + "score": -0.23551030167646542 + }, + "test": { + "rmse": 0.23422079576562096, + "mae": 0.14804303646087646, + "r2": 0.625121533870697, + "score": -0.23422079576562096 + } + }, + "time": "0:01:56.838272", + "diversity": { + "train": { + "mean_centered_corr": -0.018120070919394493, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04867309704422951 + }, + "val": { + "mean_centered_corr": -0.01935560256242752, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07195134460926056 + }, + "test": { + "mean_centered_corr": -0.019733797758817673, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09048754721879959 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_mfb-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_mfb-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_mfb-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_mfb-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..855c876e7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_mfb-greedy-heads-evaluation/0/report.json @@ -0,0 +1,245 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.22016236214854085, + "mae": 0.11783046275377274, + "r2": 0.6903293132781982, + "score": -0.22016236214854085 + }, + "val": { + "rmse": 0.23358161473727912, + "mae": 0.1258193999528885, + "r2": 0.6355578899383545, + "score": -0.23358161473727912 + }, + "test": { + "rmse": 0.2330478525093287, + "mae": 0.14592212438583374, + "r2": 0.6288667917251587, + "score": -0.2330478525093287 + } + }, + "time": "0:01:56.838272", + "diversity": { + "train": { + "mean_centered_corr": -0.018120070919394493, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04867309704422951 + }, + "val": { + "mean_centered_corr": -0.01935560256242752, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07195134460926056 + }, + "test": { + "mean_centered_corr": -0.019733797758817673, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09048754721879959 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25, + 10, + 12, + 19, + 4, + 24, + 8 + ], + "head_selection_time": "0:00:00.114671" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.0001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.0001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.0001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.0001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..a2f79f3ed --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.0001-best-head-evaluation/0/report.json @@ -0,0 +1,203 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 962, + "metrics": { + "train": { + "rmse": 0.2188904430637261, + "mae": 0.1215578094124794, + "r2": 0.6938970685005188, + "score": -0.2188904430637261 + }, + "val": { + "rmse": 0.23946739670755524, + "mae": 0.13201551139354706, + "r2": 0.6169600486755371, + "score": -0.23946739670755524 + }, + "test": { + "rmse": 0.25239608328273316, + "mae": 0.16684766113758087, + "r2": 0.5646837949752808, + "score": -0.25239608328273316 + } + }, + "time": "0:01:13.625929", + "diversity": { + "train": { + "mean_centered_corr": -0.0177121814340353, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04854941368103027 + }, + "val": { + "mean_centered_corr": -0.017364732921123505, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07598073035478592 + }, + "test": { + "mean_centered_corr": -0.012729885056614876, + "mean_pairwise_disagreement": NaN, + "member_std": 0.1117154061794281 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.014985" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.0001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.0001-evaluation/0.toml new file mode 100644 index 000000000..07fa7cfe7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.0001-evaluation/0.toml @@ -0,0 +1,41 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.0001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.0001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.0001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.0001-evaluation/0/report.json new file mode 100644 index 000000000..3bfe11f91 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.0001-evaluation/0/report.json @@ -0,0 +1,199 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 962, + "metrics": { + "train": { + "rmse": 0.2138111441702229, + "mae": 0.11529327929019928, + "r2": 0.7079383134841919, + "score": -0.2138111441702229 + }, + "val": { + "rmse": 0.23622468781228076, + "mae": 0.1262529343366623, + "r2": 0.6272635459899902, + "score": -0.23622468781228076 + }, + "test": { + "rmse": 0.23935440604862293, + "mae": 0.15282411873340607, + "r2": 0.6085084080696106, + "score": -0.23935440604862293 + } + }, + "time": "0:01:13.625929", + "diversity": { + "train": { + "mean_centered_corr": -0.0177121814340353, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04854941368103027 + }, + "val": { + "mean_centered_corr": -0.017364732921123505, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07598073035478592 + }, + "test": { + "mean_centered_corr": -0.012729885056614876, + "mean_pairwise_disagreement": NaN, + "member_std": 0.1117154061794281 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.0001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.0001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.0001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.0001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..124eabc46 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.0001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,210 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 962, + "metrics": { + "train": { + "rmse": 0.21338362874058017, + "mae": 0.11555410176515579, + "r2": 0.7091051340103149, + "score": -0.21338362874058017 + }, + "val": { + "rmse": 0.23332344829675253, + "mae": 0.1253633350133896, + "r2": 0.6363630294799805, + "score": -0.23332344829675253 + }, + "test": { + "rmse": 0.24781019077451777, + "mae": 0.16565711796283722, + "r2": 0.5803589820861816, + "score": -0.24781019077451777 + } + }, + "time": "0:01:13.625929", + "diversity": { + "train": { + "mean_centered_corr": -0.0177121814340353, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04854941368103027 + }, + "val": { + "mean_centered_corr": -0.017364732921123505, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07598073035478592 + }, + "test": { + "mean_centered_corr": -0.012729885056614876, + "mean_pairwise_disagreement": NaN, + "member_std": 0.1117154061794281 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 21, + 19, + 18, + 12, + 10, + 19, + 25 + ], + "head_selection_time": "0:00:00.129066" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..45688715d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.001-best-head-evaluation/0/report.json @@ -0,0 +1,203 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22423689267064406, + "mae": 0.12748517096042633, + "r2": 0.6787611842155457, + "score": -0.22423689267064406 + }, + "val": { + "rmse": 0.2370483756746748, + "mae": 0.13395926356315613, + "r2": 0.6246596574783325, + "score": -0.2370483756746748 + }, + "test": { + "rmse": 0.23920040609688478, + "mae": 0.14310523867607117, + "r2": 0.6090120077133179, + "score": -0.23920040609688478 + } + }, + "time": "0:01:09.057982", + "diversity": { + "train": { + "mean_centered_corr": -0.019481390714645386, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05119703337550163 + }, + "val": { + "mean_centered_corr": -0.01973012462258339, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07138136774301529 + }, + "test": { + "mean_centered_corr": -0.016405168920755386, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09870413690805435 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19 + ], + "head_selection_time": "0:00:00.015930" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.001-evaluation/0.toml new file mode 100644 index 000000000..44f3d87b5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.001-evaluation/0.toml @@ -0,0 +1,41 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.001-evaluation/0/report.json new file mode 100644 index 000000000..475cad836 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.001-evaluation/0/report.json @@ -0,0 +1,199 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22159317613989218, + "mae": 0.12286020070314407, + "r2": 0.6862912178039551, + "score": -0.22159317613989218 + }, + "val": { + "rmse": 0.2354682854519358, + "mae": 0.13157758116722107, + "r2": 0.6296467781066895, + "score": -0.2354682854519358 + }, + "test": { + "rmse": 0.24014251007009368, + "mae": 0.15741941332817078, + "r2": 0.6059260964393616, + "score": -0.24014251007009368 + } + }, + "time": "0:01:09.057982", + "diversity": { + "train": { + "mean_centered_corr": -0.019481390714645386, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05119703337550163 + }, + "val": { + "mean_centered_corr": -0.01973012462258339, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07138136774301529 + }, + "test": { + "mean_centered_corr": -0.016405168920755386, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09870413690805435 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..42d104b71 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,206 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22248795490129444, + "mae": 0.1261499673128128, + "r2": 0.6837526559829712, + "score": -0.22248795490129444 + }, + "val": { + "rmse": 0.2328613038082034, + "mae": 0.12715381383895874, + "r2": 0.6378021240234375, + "score": -0.2328613038082034 + }, + "test": { + "rmse": 0.23498425768312764, + "mae": 0.14776451885700226, + "r2": 0.6226736307144165, + "score": -0.23498425768312764 + } + }, + "time": "0:01:09.057982", + "diversity": { + "train": { + "mean_centered_corr": -0.019481390714645386, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05119703337550163 + }, + "val": { + "mean_centered_corr": -0.01973012462258339, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07138136774301529 + }, + "test": { + "mean_centered_corr": -0.016405168920755386, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09870413690805435 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19, + 21, + 4, + 24 + ], + "head_selection_time": "0:00:00.076523" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise1e-05-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise1e-05-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise1e-05-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise1e-05-best-head-evaluation/0/report.json new file mode 100644 index 000000000..0d8b2f6a0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise1e-05-best-head-evaluation/0/report.json @@ -0,0 +1,203 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22493170953564973, + "mae": 0.12546482682228088, + "r2": 0.6767673492431641, + "score": -0.22493170953564973 + }, + "val": { + "rmse": 0.2389693720816827, + "mae": 0.13471423089504242, + "r2": 0.6185516119003296, + "score": -0.2389693720816827 + }, + "test": { + "rmse": 0.24127702155259925, + "mae": 0.14682982861995697, + "r2": 0.6021938323974609, + "score": -0.24127702155259925 + } + }, + "time": "0:01:08.653647", + "diversity": { + "train": { + "mean_centered_corr": -0.018831178545951843, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048278551548719406 + }, + "val": { + "mean_centered_corr": -0.019186949357390404, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06771456450223923 + }, + "test": { + "mean_centered_corr": -0.018310436978936195, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09045998007059097 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19 + ], + "head_selection_time": "0:00:00.015389" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise1e-05-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise1e-05-evaluation/0.toml new file mode 100644 index 000000000..be03dc864 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise1e-05-evaluation/0.toml @@ -0,0 +1,41 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise1e-05-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise1e-05-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise1e-05-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise1e-05-evaluation/0/report.json new file mode 100644 index 000000000..e835cfb48 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise1e-05-evaluation/0/report.json @@ -0,0 +1,199 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2214359001653139, + "mae": 0.1211930438876152, + "r2": 0.6867363452911377, + "score": -0.2214359001653139 + }, + "val": { + "rmse": 0.23574409357673579, + "mae": 0.13180093467235565, + "r2": 0.6287786960601807, + "score": -0.23574409357673579 + }, + "test": { + "rmse": 0.2418545849757855, + "mae": 0.16086849570274353, + "r2": 0.6002870202064514, + "score": -0.2418545849757855 + } + }, + "time": "0:01:08.653647", + "diversity": { + "train": { + "mean_centered_corr": -0.018831178545951843, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048278551548719406 + }, + "val": { + "mean_centered_corr": -0.019186949357390404, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06771456450223923 + }, + "test": { + "mean_centered_corr": -0.018310436978936195, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09045998007059097 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise1e-05-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise1e-05-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise1e-05-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise1e-05-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..a475471c7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise1e-05-greedy-heads-evaluation/0/report.json @@ -0,0 +1,207 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2219257249740597, + "mae": 0.11780673265457153, + "r2": 0.6853489875793457, + "score": -0.2219257249740597 + }, + "val": { + "rmse": 0.23334154528971895, + "mae": 0.1271604299545288, + "r2": 0.636306643486023, + "score": -0.23334154528971895 + }, + "test": { + "rmse": 0.23451478286562477, + "mae": 0.1480785757303238, + "r2": 0.6241798400878906, + "score": -0.23451478286562477 + } + }, + "time": "0:01:08.653647", + "diversity": { + "train": { + "mean_centered_corr": -0.018831178545951843, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048278551548719406 + }, + "val": { + "mean_centered_corr": -0.019186949357390404, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06771456450223923 + }, + "test": { + "mean_centered_corr": -0.018310436978936195, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09045998007059097 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19, + 21, + 12, + 18, + 8 + ], + "head_selection_time": "0:00:00.088954" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.0001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.0001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.0001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.0001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..8bf3f316d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.0001-best-head-evaluation/0/report.json @@ -0,0 +1,203 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2357054301684337, + "mae": 0.12653841078281403, + "r2": 0.6450614929199219, + "score": -0.2357054301684337 + }, + "val": { + "rmse": 0.23765212789668771, + "mae": 0.13472485542297363, + "r2": 0.6227452754974365, + "score": -0.23765212789668771 + }, + "test": { + "rmse": 0.24778267915496135, + "mae": 0.16990186274051666, + "r2": 0.5804521441459656, + "score": -0.24778267915496135 + } + }, + "time": "0:01:34.341238", + "diversity": { + "train": { + "mean_centered_corr": -0.018528519198298454, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045790109783411026 + }, + "val": { + "mean_centered_corr": -0.01869109645485878, + "mean_pairwise_disagreement": NaN, + "member_std": 0.057747576385736465 + }, + "test": { + "mean_centered_corr": -0.017511537298560143, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07643643766641617 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 18 + ], + "head_selection_time": "0:00:00.015225" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.0001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.0001-evaluation/0.toml new file mode 100644 index 000000000..485d94624 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.0001-evaluation/0.toml @@ -0,0 +1,41 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.0001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.0001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.0001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.0001-evaluation/0/report.json new file mode 100644 index 000000000..6f40fec13 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.0001-evaluation/0/report.json @@ -0,0 +1,199 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2313646747453307, + "mae": 0.12118230015039444, + "r2": 0.6580142974853516, + "score": -0.2313646747453307 + }, + "val": { + "rmse": 0.2354418316334552, + "mae": 0.1258188933134079, + "r2": 0.6297299861907959, + "score": -0.2354418316334552 + }, + "test": { + "rmse": 0.23162903716515967, + "mae": 0.14173103868961334, + "r2": 0.6333720684051514, + "score": -0.23162903716515967 + } + }, + "time": "0:01:34.341238", + "diversity": { + "train": { + "mean_centered_corr": -0.018528519198298454, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045790109783411026 + }, + "val": { + "mean_centered_corr": -0.01869109645485878, + "mean_pairwise_disagreement": NaN, + "member_std": 0.057747576385736465 + }, + "test": { + "mean_centered_corr": -0.017511537298560143, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07643643766641617 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.0001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.0001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.0001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.0001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..722e24fe8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.0001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,208 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23082056510008958, + "mae": 0.12217259407043457, + "r2": 0.6596208810806274, + "score": -0.23082056510008958 + }, + "val": { + "rmse": 0.2336025622310011, + "mae": 0.12522470951080322, + "r2": 0.6354925036430359, + "score": -0.2336025622310011 + }, + "test": { + "rmse": 0.23389241285301454, + "mae": 0.14725922048091888, + "r2": 0.6261719465255737, + "score": -0.23389241285301454 + } + }, + "time": "0:01:34.341238", + "diversity": { + "train": { + "mean_centered_corr": -0.018528519198298454, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045790109783411026 + }, + "val": { + "mean_centered_corr": -0.01869109645485878, + "mean_pairwise_disagreement": NaN, + "member_std": 0.057747576385736465 + }, + "test": { + "mean_centered_corr": -0.017511537298560143, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07643643766641617 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 18, + 21, + 19, + 25, + 6, + 4 + ], + "head_selection_time": "0:00:00.101019" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..bcb05991c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.001-best-head-evaluation/0/report.json @@ -0,0 +1,203 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.22153469839383894, + "mae": 0.12104755640029907, + "r2": 0.6864567995071411, + "score": -0.22153469839383894 + }, + "val": { + "rmse": 0.23958569850307038, + "mae": 0.13397271931171417, + "r2": 0.6165815591812134, + "score": -0.23958569850307038 + }, + "test": { + "rmse": 0.2417253037065434, + "mae": 0.1469961255788803, + "r2": 0.6007142066955566, + "score": -0.2417253037065434 + } + }, + "time": "0:01:45.212537", + "diversity": { + "train": { + "mean_centered_corr": -0.018073728308081627, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047459591180086136 + }, + "val": { + "mean_centered_corr": -0.01855890266597271, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06974152475595474 + }, + "test": { + "mean_centered_corr": -0.018261540681123734, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09257550537586212 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19 + ], + "head_selection_time": "0:00:00.015445" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.001-evaluation/0.toml new file mode 100644 index 000000000..8b8f1a027 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.001-evaluation/0.toml @@ -0,0 +1,41 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.001-evaluation/0/report.json new file mode 100644 index 000000000..f138b2a28 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.001-evaluation/0/report.json @@ -0,0 +1,199 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2156155421767402, + "mae": 0.11583763360977173, + "r2": 0.7029880285263062, + "score": -0.2156155421767402 + }, + "val": { + "rmse": 0.23544592963840097, + "mae": 0.1267930269241333, + "r2": 0.6297171115875244, + "score": -0.23544592963840097 + }, + "test": { + "rmse": 0.23541433043431914, + "mae": 0.14889299869537354, + "r2": 0.6212911605834961, + "score": -0.23541433043431914 + } + }, + "time": "0:01:45.212537", + "diversity": { + "train": { + "mean_centered_corr": -0.018073728308081627, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047459591180086136 + }, + "val": { + "mean_centered_corr": -0.01855890266597271, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06974152475595474 + }, + "test": { + "mean_centered_corr": -0.018261540681123734, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09257550537586212 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..fb9bb20f1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,207 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.21704166785572343, + "mae": 0.11682380735874176, + "r2": 0.6990460157394409, + "score": -0.21704166785572343 + }, + "val": { + "rmse": 0.23389653800469046, + "mae": 0.12599465250968933, + "r2": 0.6345745325088501, + "score": -0.23389653800469046 + }, + "test": { + "rmse": 0.23994098363644276, + "mae": 0.1547209620475769, + "r2": 0.6065871715545654, + "score": -0.23994098363644276 + } + }, + "time": "0:01:45.212537", + "diversity": { + "train": { + "mean_centered_corr": -0.018073728308081627, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047459591180086136 + }, + "val": { + "mean_centered_corr": -0.01855890266597271, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06974152475595474 + }, + "test": { + "mean_centered_corr": -0.018261540681123734, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09257550537586212 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19, + 21, + 25, + 4, + 2 + ], + "head_selection_time": "0:00:00.088219" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise1e-05-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise1e-05-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise1e-05-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise1e-05-best-head-evaluation/0/report.json new file mode 100644 index 000000000..2502a80ef --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise1e-05-best-head-evaluation/0/report.json @@ -0,0 +1,203 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23729621024354838, + "mae": 0.13337191939353943, + "r2": 0.6402543783187866, + "score": -0.23729621024354838 + }, + "val": { + "rmse": 0.23804322063719607, + "mae": 0.1294262707233429, + "r2": 0.6215026378631592, + "score": -0.23804322063719607 + }, + "test": { + "rmse": 0.24414388272972984, + "mae": 0.15476657450199127, + "r2": 0.5926841497421265, + "score": -0.24414388272972984 + } + }, + "time": "0:01:34.559559", + "diversity": { + "train": { + "mean_centered_corr": -0.01910587027668953, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045081716030836105 + }, + "val": { + "mean_centered_corr": -0.01968017779290676, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0602603442966938 + }, + "test": { + "mean_centered_corr": -0.01622942090034485, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08117352426052094 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0 + ], + "head_selection_time": "0:00:00.015307" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise1e-05-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise1e-05-evaluation/0.toml new file mode 100644 index 000000000..6546774a1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise1e-05-evaluation/0.toml @@ -0,0 +1,41 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise1e-05-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise1e-05-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise1e-05-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise1e-05-evaluation/0/report.json new file mode 100644 index 000000000..3f7ae16d5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise1e-05-evaluation/0/report.json @@ -0,0 +1,199 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23182174394925567, + "mae": 0.1216563880443573, + "r2": 0.6566617488861084, + "score": -0.23182174394925567 + }, + "val": { + "rmse": 0.23604436658345962, + "mae": 0.12681564688682556, + "r2": 0.6278324127197266, + "score": -0.23604436658345962 + }, + "test": { + "rmse": 0.2328903381745147, + "mae": 0.14401739835739136, + "r2": 0.6293683052062988, + "score": -0.2328903381745147 + } + }, + "time": "0:01:34.559559", + "diversity": { + "train": { + "mean_centered_corr": -0.01910587027668953, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045081716030836105 + }, + "val": { + "mean_centered_corr": -0.01968017779290676, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0602603442966938 + }, + "test": { + "mean_centered_corr": -0.01622942090034485, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08117352426052094 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise1e-05-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise1e-05-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise1e-05-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise1e-05-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..edde87039 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise1e-05-greedy-heads-evaluation/0/report.json @@ -0,0 +1,211 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23216803497367422, + "mae": 0.12254077196121216, + "r2": 0.6556352376937866, + "score": -0.23216803497367422 + }, + "val": { + "rmse": 0.2335869733918295, + "mae": 0.12630410492420197, + "r2": 0.6355411410331726, + "score": -0.2335869733918295 + }, + "test": { + "rmse": 0.2353936312104005, + "mae": 0.15104448795318604, + "r2": 0.621357798576355, + "score": -0.2353936312104005 + } + }, + "time": "0:01:34.559559", + "diversity": { + "train": { + "mean_centered_corr": -0.01910587027668953, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045081716030836105 + }, + "val": { + "mean_centered_corr": -0.01968017779290676, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0602603442966938 + }, + "test": { + "mean_centered_corr": -0.01622942090034485, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08117352426052094 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0, + 4, + 18, + 16, + 12, + 25, + 21, + 18, + 19 + ], + "head_selection_time": "0:00:00.145132" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.0001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.0001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.0001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.0001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..474cdf3b1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.0001-best-head-evaluation/0/report.json @@ -0,0 +1,203 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 6702952, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23628005807891533, + "mae": 0.12606193125247955, + "r2": 0.6433287858963013, + "score": -0.23628005807891533 + }, + "val": { + "rmse": 0.23612384794902308, + "mae": 0.13170304894447327, + "r2": 0.6275817155838013, + "score": -0.23612384794902308 + }, + "test": { + "rmse": 0.25131993238610145, + "mae": 0.1727743297815323, + "r2": 0.5683881044387817, + "score": -0.25131993238610145 + } + }, + "time": "0:02:37.152304", + "diversity": { + "train": { + "mean_centered_corr": -0.018440110608935356, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04351954534649849 + }, + "val": { + "mean_centered_corr": -0.019317032769322395, + "mean_pairwise_disagreement": NaN, + "member_std": 0.059053633362054825 + }, + "test": { + "mean_centered_corr": -0.019231578335165977, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07535656541585922 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 18 + ], + "head_selection_time": "0:00:00.015278" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.0001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.0001-evaluation/0.toml new file mode 100644 index 000000000..766daf18b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.0001-evaluation/0.toml @@ -0,0 +1,41 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 8 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.0001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.0001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.0001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.0001-evaluation/0/report.json new file mode 100644 index 000000000..73100e216 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.0001-evaluation/0/report.json @@ -0,0 +1,199 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 6702952, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2319284940460109, + "mae": 0.12257123738527298, + "r2": 0.6563454270362854, + "score": -0.2319284940460109 + }, + "val": { + "rmse": 0.23560034396487695, + "mae": 0.12617181241512299, + "r2": 0.6292312741279602, + "score": -0.23560034396487695 + }, + "test": { + "rmse": 0.2314192197875428, + "mae": 0.1411750614643097, + "r2": 0.6340359449386597, + "score": -0.2314192197875428 + } + }, + "time": "0:02:37.152304", + "diversity": { + "train": { + "mean_centered_corr": -0.018440110608935356, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04351954534649849 + }, + "val": { + "mean_centered_corr": -0.019317032769322395, + "mean_pairwise_disagreement": NaN, + "member_std": 0.059053633362054825 + }, + "test": { + "mean_centered_corr": -0.019231578335165977, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07535656541585922 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.0001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.0001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.0001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.0001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..0dc9b308b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.0001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,211 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 6702952, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23160647965627332, + "mae": 0.12231546640396118, + "r2": 0.6572990417480469, + "score": -0.23160647965627332 + }, + "val": { + "rmse": 0.23331262295778735, + "mae": 0.12554694712162018, + "r2": 0.6363967657089233, + "score": -0.23331262295778735 + }, + "test": { + "rmse": 0.2320760028721284, + "mae": 0.14371301233768463, + "r2": 0.6319557428359985, + "score": -0.2320760028721284 + } + }, + "time": "0:02:37.152304", + "diversity": { + "train": { + "mean_centered_corr": -0.018440110608935356, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04351954534649849 + }, + "val": { + "mean_centered_corr": -0.019317032769322395, + "mean_pairwise_disagreement": NaN, + "member_std": 0.059053633362054825 + }, + "test": { + "mean_centered_corr": -0.019231578335165977, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07535656541585922 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 18, + 25, + 19, + 28, + 31, + 21, + 6, + 25, + 4 + ], + "head_selection_time": "0:00:00.145454" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..e2e44ae71 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.001-best-head-evaluation/0/report.json @@ -0,0 +1,203 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 6702952, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2211915746692412, + "mae": 0.12369639426469803, + "r2": 0.687427282333374, + "score": -0.2211915746692412 + }, + "val": { + "rmse": 0.239120553017687, + "mae": 0.13442911207675934, + "r2": 0.6180688738822937, + "score": -0.239120553017687 + }, + "test": { + "rmse": 0.23826554512424344, + "mae": 0.14826232194900513, + "r2": 0.6120622158050537, + "score": -0.23826554512424344 + } + }, + "time": "0:02:54.576470", + "diversity": { + "train": { + "mean_centered_corr": -0.018232719972729683, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045184310525655746 + }, + "val": { + "mean_centered_corr": -0.019308097660541534, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06649133563041687 + }, + "test": { + "mean_centered_corr": -0.017781618982553482, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09422237426042557 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015104" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.001-evaluation/0.toml new file mode 100644 index 000000000..cfb125e1a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.001-evaluation/0.toml @@ -0,0 +1,41 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 8 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.001-evaluation/0/report.json new file mode 100644 index 000000000..46253077e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.001-evaluation/0/report.json @@ -0,0 +1,199 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 6702952, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.21571915616347168, + "mae": 0.11579738557338715, + "r2": 0.7027024626731873, + "score": -0.21571915616347168 + }, + "val": { + "rmse": 0.23625085686725053, + "mae": 0.12631435692310333, + "r2": 0.6271809935569763, + "score": -0.23625085686725053 + }, + "test": { + "rmse": 0.2355661245169954, + "mae": 0.147227481007576, + "r2": 0.620802640914917, + "score": -0.2355661245169954 + } + }, + "time": "0:02:54.576470", + "diversity": { + "train": { + "mean_centered_corr": -0.018232719972729683, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045184310525655746 + }, + "val": { + "mean_centered_corr": -0.019308097660541534, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06649133563041687 + }, + "test": { + "mean_centered_corr": -0.017781618982553482, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09422237426042557 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..faaf6ce89 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,212 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 6702952, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.21646584334087177, + "mae": 0.11596201360225677, + "r2": 0.7006407976150513, + "score": -0.21646584334087177 + }, + "val": { + "rmse": 0.23353637226653048, + "mae": 0.1261347234249115, + "r2": 0.6356990337371826, + "score": -0.23353637226653048 + }, + "test": { + "rmse": 0.23842460978700958, + "mae": 0.1537153124809265, + "r2": 0.6115440726280212, + "score": -0.23842460978700958 + } + }, + "time": "0:02:54.576470", + "diversity": { + "train": { + "mean_centered_corr": -0.018232719972729683, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045184310525655746 + }, + "val": { + "mean_centered_corr": -0.019308097660541534, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06649133563041687 + }, + "test": { + "mean_centered_corr": -0.017781618982553482, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09422237426042557 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 19, + 12, + 18, + 2, + 21, + 25, + 19, + 12, + 10 + ], + "head_selection_time": "0:00:00.158560" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise1e-05-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise1e-05-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise1e-05-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise1e-05-best-head-evaluation/0/report.json new file mode 100644 index 000000000..8d7959fac --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise1e-05-best-head-evaluation/0/report.json @@ -0,0 +1,203 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 6702952, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22504498078493876, + "mae": 0.12070199847221375, + "r2": 0.6764416694641113, + "score": -0.22504498078493876 + }, + "val": { + "rmse": 0.23899585630204528, + "mae": 0.13757818937301636, + "r2": 0.6184670925140381, + "score": -0.23899585630204528 + }, + "test": { + "rmse": 0.25744912519923097, + "mae": 0.18095265328884125, + "r2": 0.5470789670944214, + "score": -0.25744912519923097 + } + }, + "time": "0:02:49.448996", + "diversity": { + "train": { + "mean_centered_corr": -0.018503453582525253, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04646626114845276 + }, + "val": { + "mean_centered_corr": -0.018042705953121185, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06875067204236984 + }, + "test": { + "mean_centered_corr": -0.016347019001841545, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09534233063459396 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 18 + ], + "head_selection_time": "0:00:00.015265" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise1e-05-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise1e-05-evaluation/0.toml new file mode 100644 index 000000000..82c69bc53 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise1e-05-evaluation/0.toml @@ -0,0 +1,41 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 8 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise1e-05-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise1e-05-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise1e-05-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise1e-05-evaluation/0/report.json new file mode 100644 index 000000000..24a7a971e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise1e-05-evaluation/0/report.json @@ -0,0 +1,199 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 6702952, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.21933724563493878, + "mae": 0.11967557668685913, + "r2": 0.6926461458206177, + "score": -0.21933724563493878 + }, + "val": { + "rmse": 0.23553038961691936, + "mae": 0.13043814897537231, + "r2": 0.6294513940811157, + "score": -0.23553038961691936 + }, + "test": { + "rmse": 0.2411268296110548, + "mae": 0.15842990577220917, + "r2": 0.6026889085769653, + "score": -0.2411268296110548 + } + }, + "time": "0:02:49.448996", + "diversity": { + "train": { + "mean_centered_corr": -0.018503453582525253, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04646626114845276 + }, + "val": { + "mean_centered_corr": -0.018042705953121185, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06875067204236984 + }, + "test": { + "mean_centered_corr": -0.016347019001841545, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09534233063459396 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise1e-05-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise1e-05-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise1e-05-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise1e-05-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..1a1c1c437 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise1e-05-greedy-heads-evaluation/0/report.json @@ -0,0 +1,210 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 6702952, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2187223246642418, + "mae": 0.1176944449543953, + "r2": 0.6943670511245728, + "score": -0.2187223246642418 + }, + "val": { + "rmse": 0.23346652550110514, + "mae": 0.1266631931066513, + "r2": 0.6359169483184814, + "score": -0.23346652550110514 + }, + "test": { + "rmse": 0.24465339979644968, + "mae": 0.16457043588161469, + "r2": 0.5909823179244995, + "score": -0.24465339979644968 + } + }, + "time": "0:02:49.448996", + "diversity": { + "train": { + "mean_centered_corr": -0.018503453582525253, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04646626114845276 + }, + "val": { + "mean_centered_corr": -0.018042705953121185, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06875067204236984 + }, + "test": { + "mean_centered_corr": -0.016347019001841545, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09534233063459396 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 18, + 19, + 4, + 27, + 21, + 12, + 24, + 1 + ], + "head_selection_time": "0:00:00.130591" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/rescue_selected_configs.csv b/tabm_integrated/paper/exp/rescue_selected_configs.csv new file mode 100644 index 000000000..8278598c8 --- /dev/null +++ b/tabm_integrated/paper/exp/rescue_selected_configs.csv @@ -0,0 +1,6 @@ +dataset,module,final_variant,source_variant,source_wave,inference_mode,validation_metric,matched_validation_baseline,validation_delta,validation_percent_delta,validation_status,validation_score,source_config_path,seed0_result_path,confirm_3seed +delivery-eta,RLA,best_rla_only,deliv_rla_r4_n0p0001_lr0p5_fr0,rescue_delivery_individual,best-head,0.5572652691682626,0.5583881453279098,0.0011228761596472348,0.20109240660684746,validation_win,-0.5572652691682626,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0-best-head-evaluation/0,True +delivery-eta,ESAM,best_esam_only,deliv_esam_rho0p001_sf0p25_adapterf,rescue_delivery_individual,mean,0.5558269510040955,0.5558749906446844,4.8039640588859456e-05,0.008642166205956623,validation_win,-0.5558269510040955,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adapterf-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adapterf-evaluation/0,True +delivery-eta,MFB,best_mfb_only,deliv_mfb_k0p975_sf0p5_numerical_only,rescue_delivery_individual,mean,0.5558048331442785,0.5558749906446844,7.015750040584301e-05,0.012621093157020213,validation_win,-0.5558048331442785,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_numerical_only-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_numerical_only-evaluation/0,True +delivery-eta,CF-FISD,best_cf_fisd_only,deliv_cf_l0p005_consensus_raw,rescue_delivery_individual,greedy-heads,0.5553348954063525,0.5553348954063525,0.0,0.0,close_no_validation_win,-0.5553348954063525,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_consensus_raw-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_consensus_raw-greedy-heads-evaluation/0,True +delivery-eta,combined,best_combined,deliv_comb_rla_cf,rescue_delivery_combined,best-head,0.5572652691682626,0.5583881453279098,0.0011228761596472348,0.20109240660684746,validation_win,-0.5572652691682626,exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_cf-evaluation/0.toml,exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_cf-best-head-evaluation/0,True diff --git a/tabm_integrated/paper/exp/rescue_targets.csv b/tabm_integrated/paper/exp/rescue_targets.csv new file mode 100644 index 000000000..a5c900274 --- /dev/null +++ b/tabm_integrated/paper/exp/rescue_targets.csv @@ -0,0 +1,6 @@ +priority,dataset,module,final_variant,current_source_variant,current_inference_mode,metric,direction,matched_baseline_test_mean,current_test_mean,current_absolute_delta,current_percent_delta,current_status,rescue_reason +0,delivery-eta,RLA,best_rla_only,rla_rank2_noise0.001,greedy-heads,RMSE,lower,0.5511744515458483,0.5514065614560619,-0.00023210991021360705,-0.04211187756664361,loss,matched_baseline_loss +0,delivery-eta,ESAM,best_esam_only,esam_only,greedy-heads,RMSE,lower,0.5511744515458483,0.5511479242060323,2.652733981600708e-05,0.004812875441088992,weak_win,delivery_eta_priority +0,delivery-eta,MFB,best_mfb_only,mfb_keep0.7,greedy-heads,RMSE,lower,0.5511744515458483,0.551366165184366,-0.00019171363851766543,-0.03478275126504446,loss,matched_baseline_loss +0,delivery-eta,CF-FISD,best_cf_fisd_only,cf_fisd_lambda0.05,greedy-heads,RMSE,lower,0.5511744515458483,0.5524340881807381,-0.0012596366348898158,-0.22853683282252707,loss,matched_baseline_loss +0,delivery-eta,combined,best_combined,rla_esam,greedy-heads,RMSE,lower,0.5511744515458483,0.5523362750771902,-0.0011618235313418657,-0.21079052704336418,loss,matched_baseline_loss diff --git a/tabm_integrated/paper/exp/sberbank_combined_rescue_confirmed.csv b/tabm_integrated/paper/exp/sberbank_combined_rescue_confirmed.csv new file mode 100644 index 000000000..6ac8242a1 --- /dev/null +++ b/tabm_integrated/paper/exp/sberbank_combined_rescue_confirmed.csv @@ -0,0 +1,3 @@ +wave,dataset,variant,seed,metric,metric_key,direction,validation_metric,validation_score,test_metric,inference_mode,config_path,result_path,failure,best_step,matched_validation_baseline,validation_delta,validation_percent_delta,validation_status,mean,std,baseline_mean,baseline_std,absolute_delta,percent_delta,status +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001,0,RMSE,rmse,lower,0.23560357747729638,-0.23560357747729638,0.2389191702722735,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-best-head-evaluation/0,False,666,0.23955454428788905,0.0039509668105926765,1.6492973749830142,validation_win,0.24507966333691242,0.008112050773101143,0.2502002926581795,0.013600655537586393,0.005120629321267084,2.0466120430413817,weak_win +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001,0,RMSE,rmse,lower,0.23560357747729638,-0.23560357747729638,0.2389191702722735,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-best-head-evaluation/0,False,666,0.23955454428788905,0.0039509668105926765,1.6492973749830142,validation_win,0.24507966333691242,0.008112050773101143,0.2502002926581795,0.013600655537586393,0.005120629321267084,2.0466120430413817,weak_win diff --git a/tabm_integrated/paper/exp/sberbank_combined_rescue_selected.csv b/tabm_integrated/paper/exp/sberbank_combined_rescue_selected.csv new file mode 100644 index 000000000..bfc7a42a6 --- /dev/null +++ b/tabm_integrated/paper/exp/sberbank_combined_rescue_selected.csv @@ -0,0 +1,21 @@ +wave,dataset,variant,seed,metric,metric_key,direction,validation_metric,validation_score,test_metric,inference_mode,config_path,result_path,failure,best_step,matched_validation_baseline,validation_delta,validation_percent_delta,validation_status +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001,0,RMSE,rmse,lower,0.23560357747729638,-0.23560357747729638,0.2389191702722735,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-best-head-evaluation/0,False,666,0.23955454428788905,0.0039509668105926765,1.6492973749830142,validation_win +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001,0,RMSE,rmse,lower,0.23560357747729638,-0.23560357747729638,0.2389191702722735,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-best-head-evaluation/0,False,666,0.23955454428788905,0.0039509668105926765,1.6492973749830142,validation_win +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p001,0,RMSE,rmse,lower,0.23560357747729638,-0.23560357747729638,0.2389191702722735,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p001-best-head-evaluation/0,False,666,0.23955454428788905,0.0039509668105926765,1.6492973749830142,validation_win +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p001,0,RMSE,rmse,lower,0.2357278561917395,-0.2357278561917395,0.24177902137278479,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p001-best-head-evaluation/0,False,666,0.23955454428788905,0.0038266880961495553,1.5974182863134347,validation_win +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p005,0,RMSE,rmse,lower,0.2360115531890879,-0.2360115531890879,0.23473422019733492,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p005-best-head-evaluation/0,False,666,0.23955454428788905,0.003542991098801157,1.4789913960234888,validation_win +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p001,0,RMSE,rmse,lower,0.23604388522704514,-0.23604388522704514,0.2413116661749965,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p001-best-head-evaluation/0,False,666,0.23955454428788905,0.0035106590608439125,1.4654946627207013,validation_win +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p005,0,RMSE,rmse,lower,0.23613982935624828,-0.23613982935624828,0.23869431760070778,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p005-best-head-evaluation/0,False,666,0.23955454428788905,0.0034147149316407777,1.4254436048339294,validation_win +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p005,0,RMSE,rmse,lower,0.2361938000242818,-0.2361938000242818,0.23968299921179284,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p005-best-head-evaluation/0,False,666,0.23955454428788905,0.003360744263607246,1.4029140100838204,validation_win +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p01,0,RMSE,rmse,lower,0.23633714926648194,-0.23633714926648194,0.23801691214143877,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p01-best-head-evaluation/0,False,666,0.23955454428788905,0.00321739502140711,1.3430740923622584,validation_win +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p01,0,RMSE,rmse,lower,0.23637143045682502,-0.23637143045682502,0.2444339973181191,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p01-best-head-evaluation/0,False,666,0.23955454428788905,0.0031831138310640383,1.3287637020313308,validation_win +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p01,0,RMSE,rmse,lower,0.236445130001989,-0.236445130001989,0.2384320235467145,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p01-best-head-evaluation/0,False,666,0.23955454428788905,0.0031094142859000584,1.2979984559020776,validation_win +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p01,0,RMSE,rmse,lower,0.23656618722639788,-0.23656618722639788,0.2443526301182123,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p01-best-head-evaluation/0,False,666,0.23955454428788905,0.0029883570614911703,1.2474641507530149,validation_win +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p01,0,RMSE,rmse,lower,0.23656618722639788,-0.23656618722639788,0.2443526301182123,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p01-best-head-evaluation/0,False,666,0.23955454428788905,0.0029883570614911703,1.2474641507530149,validation_win +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p01,0,RMSE,rmse,lower,0.23656618722639788,-0.23656618722639788,0.2443526301182123,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p01-best-head-evaluation/0,False,666,0.23955454428788905,0.0029883570614911703,1.2474641507530149,validation_win +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p005,0,RMSE,rmse,lower,0.23674080812018197,-0.23674080812018197,0.24022462096222602,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p005-best-head-evaluation/0,False,666,0.23955454428788905,0.0028137361677070816,1.1745701489701748,validation_win +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0p001_rho0p0005,0,RMSE,rmse,lower,0.23686825686043878,-0.23686825686043878,0.24350702848855615,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0005-best-head-evaluation/0,False,666,0.23955454428788905,0.002686287427450279,1.1213677600797187,validation_win +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p005,0,RMSE,rmse,lower,0.2369960693297996,-0.2369960693297996,0.2449247661219165,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p005-best-head-evaluation/0,False,666,0.23955454428788905,0.002558474958089446,1.0680135355790839,validation_win +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p001,0,RMSE,rmse,lower,0.237046057654127,-0.237046057654127,0.2530936074511429,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p001-best-head-evaluation/0,False,888,0.23955454428788905,0.002508486633762047,1.0471463362212103,validation_win +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0p0001_rho0p00025,0,RMSE,rmse,lower,0.23704860354665752,-0.23704860354665752,0.23874426241915847,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p00025-best-head-evaluation/0,False,814,0.23955454428788905,0.0025059407412315327,1.046083575112636,validation_win +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p005,0,RMSE,rmse,lower,0.23707190033649148,-0.23707190033649148,0.2397359546762736,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p005-best-head-evaluation/0,False,666,0.23955454428788905,0.0024826439513975784,1.0363585290262813,validation_win diff --git a/tabm_integrated/paper/exp/selected_integrated_configs.csv b/tabm_integrated/paper/exp/selected_integrated_configs.csv new file mode 100644 index 000000000..c60733aac --- /dev/null +++ b/tabm_integrated/paper/exp/selected_integrated_configs.csv @@ -0,0 +1,31 @@ +dataset,final_variant,source_variant,source_wave,inference_mode,validation_metric,validation_score,source_config_path,matched_validation_baseline,validation_delta,validation_percent_delta,validation_status +sberbank-housing,baseline_plr,baseline_plr,smoke,mean,,,exp/integrated/smoke/sberbank-housing/baseline_plr-evaluation/0.toml,,,, +sberbank-housing,best_rla_only,rla_rank2_noise0.001,sweeps,greedy-heads,0.2328613038082034,-0.2328613038082034,exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.001-evaluation/0.toml,,,, +sberbank-housing,best_esam_only,esam_rho0.0025,sweeps,greedy-heads,0.23295802298295287,-0.23295802298295287,exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-evaluation/0.toml,,,, +sberbank-housing,best_mfb_only,mfb_keep0.8,sweeps,greedy-heads,0.23237596587359466,-0.23237596587359466,exp/integrated/sweeps/sberbank-housing/mfb_keep0.8-evaluation/0.toml,,,, +sberbank-housing,best_cf_fisd_only,cf_fisd_only,smoke,greedy-heads,0.23327114512588593,-0.23327114512588593,exp/integrated/smoke/sberbank-housing/cf_fisd_only-evaluation/0.toml,,,, +sberbank-housing,best_combined,sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001,rescue_sberbank_combined,best-head,0.23560357747729638,-0.23560357747729638,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/0.toml,0.23955454428788905,0.0039509668105926765,1.6492973749830142,validation_win +ecom-offers,baseline_plr,baseline_plr,smoke,mean,,,exp/integrated/smoke/ecom-offers/baseline_plr-evaluation/0.toml,,,, +ecom-offers,best_rla_only,rla_rank2_noise0.0001,sweeps,best-head,0.6396541901648656,0.6396541901648656,exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.0001-evaluation/0.toml,,,, +ecom-offers,best_esam_only,esam_rho0.001,sweeps,greedy-heads,0.641845068859757,0.641845068859757,exp/integrated/sweeps/ecom-offers/esam_rho0.001-evaluation/0.toml,,,, +ecom-offers,best_mfb_only,mfb_keep0.7,sweeps,best-head,0.6547032702847417,0.6547032702847417,exp/integrated/sweeps/ecom-offers/mfb_keep0.7-evaluation/0.toml,,,, +ecom-offers,best_cf_fisd_only,cf_fisd_lambda0.05,sweeps,greedy-heads,0.6419149054445072,0.6419149054445072,exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.05-evaluation/0.toml,,,, +ecom-offers,best_combined,mfb_cf_fisd,sweeps,greedy-heads,0.6506783465179156,0.6506783465179156,exp/integrated/sweeps/ecom-offers/mfb_cf_fisd-evaluation/0.toml,,,, +homesite-insurance,baseline_plr,baseline_plr,smoke,mean,,,exp/integrated/smoke/homesite-insurance/baseline_plr-evaluation/0.toml,,,, +homesite-insurance,best_rla_only,rla_rank4_noise1e-05,sweeps,greedy-heads,0.9591329763748824,0.9591329763748824,exp/integrated/sweeps/homesite-insurance/rla_rank4_noise1e-05-evaluation/0.toml,,,, +homesite-insurance,best_esam_only,esam_only,smoke,greedy-heads,0.958991133281215,0.958991133281215,exp/integrated/smoke/homesite-insurance/esam_only-evaluation/0.toml,,,, +homesite-insurance,best_mfb_only,mfb_keep0.95,sweeps,greedy-heads,0.9592957095194116,0.9592957095194116,exp/integrated/sweeps/homesite-insurance/mfb_keep0.95-evaluation/0.toml,,,, +homesite-insurance,best_cf_fisd_only,cf_fisd_only,smoke,greedy-heads,0.9594844321379927,0.9594844321379927,exp/integrated/smoke/homesite-insurance/cf_fisd_only-evaluation/0.toml,,,, +homesite-insurance,best_combined,mfb_cf_fisd,sweeps,greedy-heads,0.9597570226838343,0.9597570226838343,exp/integrated/sweeps/homesite-insurance/mfb_cf_fisd-evaluation/0.toml,,,, +cooking-time,baseline_plr,baseline_plr,smoke,mean,,,exp/integrated/smoke/cooking-time/baseline_plr-evaluation/0.toml,,,, +cooking-time,best_rla_only,rla_rank4_noise0.001,sweeps,greedy-heads,0.4624107384246987,-0.4624107384246987,exp/integrated/sweeps/cooking-time/rla_rank4_noise0.001-evaluation/0.toml,,,, +cooking-time,best_esam_only,esam_rho0.0025,sweeps,greedy-heads,0.46246732196836127,-0.46246732196836127,exp/integrated/sweeps/cooking-time/esam_rho0.0025-evaluation/0.toml,,,, +cooking-time,best_mfb_only,mfb_keep0.8,sweeps,greedy-heads,0.4624822561623469,-0.4624822561623469,exp/integrated/sweeps/cooking-time/mfb_keep0.8-evaluation/0.toml,,,, +cooking-time,best_cf_fisd_only,cf_fisd_lambda0.05,sweeps,greedy-heads,0.4628482268025946,-0.4628482268025946,exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.05-evaluation/0.toml,,,, +cooking-time,best_combined,rla_esam,sweeps,greedy-heads,0.4624234187671886,-0.4624234187671886,exp/integrated/sweeps/cooking-time/rla_esam-evaluation/0.toml,,,, +delivery-eta,baseline_plr,baseline_plr,smoke,mean,,,exp/integrated/smoke/delivery-eta/baseline_plr-evaluation/0.toml,,,, +delivery-eta,best_rla_only,deliv_rla_r4_n0p0001_lr0p5_fr0,rescue_delivery_individual,best-head,0.5572652691682626,-0.5572652691682626,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0-evaluation/0.toml,0.5583881453279098,0.0011228761596472348,0.20109240660684746,validation_win +delivery-eta,best_esam_only,deliv_esam_rho0p001_sf0p25_adapterf,rescue_delivery_individual,mean,0.5558269510040955,-0.5558269510040955,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adapterf-evaluation/0.toml,0.5558749906446844,4.8039640588859456e-05,0.008642166205956623,validation_win +delivery-eta,best_mfb_only,deliv_mfb_k0p975_sf0p5_numerical_only,rescue_delivery_individual,mean,0.5558048331442785,-0.5558048331442785,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_numerical_only-evaluation/0.toml,0.5558749906446844,7.015750040584301e-05,0.012621093157020213,validation_win +delivery-eta,best_cf_fisd_only,deliv_cf_l0p005_consensus_raw,rescue_delivery_individual,greedy-heads,0.5553348954063525,-0.5553348954063525,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_consensus_raw-evaluation/0.toml,0.5553348954063525,0.0,0.0,close_no_validation_win +delivery-eta,best_combined,deliv_comb_rla_cf,rescue_delivery_combined,best-head,0.5572652691682626,-0.5572652691682626,exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_cf-evaluation/0.toml,0.5583881453279098,0.0011228761596472348,0.20109240660684746,validation_win diff --git a/tabm_integrated/paper/lib/__init__.py b/tabm_integrated/paper/lib/__init__.py new file mode 100644 index 000000000..f3e5e9015 --- /dev/null +++ b/tabm_integrated/paper/lib/__init__.py @@ -0,0 +1,5 @@ +from __future__ import annotations +from . import env as _env # Check the current working directory. +from .util import * # noqa: F403 + +del _env diff --git a/tabm_integrated/paper/lib/cf_fisd.py b/tabm_integrated/paper/lib/cf_fisd.py new file mode 100644 index 000000000..811a725f2 --- /dev/null +++ b/tabm_integrated/paper/lib/cf_fisd.py @@ -0,0 +1,125 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Mapping, Sequence + +import numpy as np +import torch +import torch.nn.functional as F +from torch import Tensor + +VARIANTS = ('softmax', 'l1norm', 'raw') +DEFAULT_TEACHER_NAMES: tuple[str, ...] = ('xgb', 'lgbm', 'cat') + + +def chunk_aggregate_r1(r1: Tensor, d_features: Sequence[int]) -> Tensor: + if r1.dim() != 2: + raise ValueError(f'r1 must be 2D, got shape {tuple(r1.shape)}') + expected = int(sum(d_features)) + if r1.shape[-1] != expected: + raise ValueError( + f'r1 last dim {r1.shape[-1]} != sum(d_features)={expected}' + ) + abs_r = r1.abs() + chunks = abs_r.split(list(d_features), dim=-1) + return torch.stack([c.sum(dim=-1) for c in chunks], dim=-1) + + +def cf_fisd_loss( + r1: Tensor, + teacher_importances: Mapping[str, Tensor], + member_groups: Mapping[str, Sequence[int]], + variant: str, + d_features: Sequence[int], +) -> Tensor: + if variant not in VARIANTS: + raise ValueError(f'unknown variant: {variant!r}; expected one of {VARIANTS}') + if not member_groups: + return r1.new_zeros(()) + + r1_mag = chunk_aggregate_r1(r1, d_features) + losses: list[Tensor] = [] + for teacher_name, member_ids in member_groups.items(): + if not member_ids: + continue + if teacher_name not in teacher_importances: + raise KeyError(f'teacher_importances missing {teacher_name!r}') + t_imp = teacher_importances[teacher_name].to(r1.device, dtype=r1_mag.dtype) + for m in member_ids: + r_m = r1_mag[int(m)] + if variant == 'softmax': + pred = F.softmax(r_m, dim=-1) + tgt = F.softmax(t_imp, dim=-1) + elif variant == 'l1norm': + pred = r_m / (r_m.sum() + 1e-8) + tgt = t_imp / (t_imp.sum() + 1e-8) + else: + tgt = t_imp / (t_imp.sum() + 1e-8) * r_m.sum().detach() + pred = r_m + losses.append(F.mse_loss(pred, tgt, reduction='mean')) + if not losses: + return r1.new_zeros(()) + return torch.stack(losses).mean() + + +def alignment_cosine( + r1: Tensor, + teacher_imp: Tensor, + member_ids: Sequence[int], + d_features: Sequence[int], +) -> Tensor: + r1_soft = F.softmax(chunk_aggregate_r1(r1, d_features), dim=-1) + t_soft = F.softmax(teacher_imp.to(r1.device, dtype=r1_soft.dtype), dim=-1) + ids = list(member_ids) + return F.cosine_similarity( + r1_soft[ids], t_soft.unsqueeze(0).expand(len(ids), -1), dim=-1 + ) + + +def load_teacher_importances( + teacher_dir: str | Path, + dataset_name: str, + n_features: int, + teacher_names: Sequence[str] = DEFAULT_TEACHER_NAMES, +) -> dict[str, Tensor]: + out: dict[str, Tensor] = {} + root = Path(teacher_dir) + for name in teacher_names: + f = root / f'{name}.npy' + if not f.exists(): + f_alt = root / f'{dataset_name}_{name}.npy' + if f_alt.exists(): + f = f_alt + else: + raise FileNotFoundError(f) + v = np.load(f).astype(np.float32) + if v.shape != (n_features,): + raise ValueError( + f'{f}: expected shape ({n_features},), got {v.shape}' + ) + v = np.clip(v, 0.0, None) + s = float(v.sum()) + if s > 0.0: + v = v / s + else: + v = np.full_like(v, 1.0 / n_features) + out[name] = torch.from_numpy(v) + return out + + +def default_member_groups( + k: int, + teacher_names: Sequence[str] = DEFAULT_TEACHER_NAMES, +) -> dict[str, list[int]]: + n = len(teacher_names) + if n == 0: + raise ValueError('teacher_names must not be empty') + base = k // n + extra = k - base * n + groups: dict[str, list[int]] = {} + cursor = 0 + for i, name in enumerate(teacher_names): + size = base + (1 if i < extra else 0) + groups[name] = list(range(cursor, cursor + size)) + cursor += size + return groups diff --git a/tabm_integrated/paper/lib/data.py b/tabm_integrated/paper/lib/data.py new file mode 100644 index 000000000..22562fb44 --- /dev/null +++ b/tabm_integrated/paper/lib/data.py @@ -0,0 +1,397 @@ +from __future__ import annotations +import enum +import hashlib +import json +import pickle +from collections.abc import Iterable +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Generic, TypeVar, cast + +import numpy as np +import sklearn.preprocessing +import torch +from loguru import logger +from torch import Tensor + +from . import env +from .metrics import calculate_metrics as calculate_metrics_ +from .util import DataKey, PartKey, PredictionType, Score, TaskType + +_SCORE_SHOULD_BE_MAXIMIZED = { + Score.ACCURACY: True, + Score.CROSS_ENTROPY: False, + Score.MAE: False, + Score.R2: True, + Score.RMSE: False, + Score.ROC_AUC: True, +} + + +@dataclass(frozen=True) +class Task: + labels: dict[PartKey, np.ndarray] + type_: TaskType + score: Score + + @classmethod + def from_dir(cls, path: str | Path) -> 'Task': + path = Path(path).resolve() + info = json.loads(path.joinpath('info.json').read_text()) + task_type = TaskType(info['task_type']) + score = info.get('score') + if score is None: + score = { + TaskType.BINCLASS: Score.ACCURACY, + TaskType.MULTICLASS: Score.ACCURACY, + TaskType.REGRESSION: Score.RMSE, + }[task_type] + else: + score = Score(score) + return Task( + { + part: np.load(path / f'Y_{part}.npy') + for part in ['train', 'val', 'test'] + }, + task_type, + score, + ) + + def __post_init__(self): + assert isinstance(self.type_, TaskType) + assert isinstance(self.score, Score) + if self.is_regression: + assert all( + value.dtype in (np.dtype('float32'), np.dtype('float64')) + for value in self.labels.values() + ), 'Regression labels must have dtype=float32' + for key in self.labels: + self.labels[key] = self.labels[key].astype('float32') + + @property + def is_regression(self) -> bool: + return self.type_ == TaskType.REGRESSION + + @property + def is_binclass(self) -> bool: + return self.type_ == TaskType.BINCLASS + + @property + def is_multiclass(self) -> bool: + return self.type_ == TaskType.MULTICLASS + + @property + def is_classification(self) -> bool: + return self.is_binclass or self.is_multiclass + + def compute_n_classes(self) -> int: + assert self.is_binclass or self.is_classification + return len(np.unique(self.labels['train'])) + + def try_compute_n_classes(self) -> None | int: + return None if self.is_regression else self.compute_n_classes() + + def calculate_metrics( + self, + predictions: dict[PartKey, np.ndarray], + prediction_type: str | PredictionType, + ) -> dict[PartKey, Any]: + metrics = { + part: calculate_metrics_( + self.labels[part], predictions[part], self.type_, prediction_type + ) + for part in predictions + } + for part_metrics in metrics.values(): + part_metrics['score'] = ( + 1.0 if _SCORE_SHOULD_BE_MAXIMIZED[self.score] else -1.0 + ) * part_metrics[self.score.value] + return metrics # type: ignore[code] + + +def load_data(path: str | Path) -> dict[DataKey, dict[PartKey, np.ndarray]]: + path = Path(path).resolve() + return { # type: ignore[code] + key.lower(): { + part: np.load(path / f'{key}_{part}.npy', allow_pickle=True) + for part in ['train', 'val', 'test'] + } + for key in ['X_num', 'X_bin', 'X_cat', 'Y'] + if path.joinpath(f'{key}_train.npy').exists() + } + + +T = TypeVar('T', np.ndarray, Tensor) + + +@dataclass +class Dataset(Generic[T]): + """Dataset = Data + Task + simple methods for convenience. + + The task is stored separately to ensure that the original labels never change. + """ + + data: dict[DataKey, dict[PartKey, T]] + task: Task + + @classmethod + def from_dir(cls, path: str | Path) -> 'Dataset[np.ndarray]': + return Dataset(load_data(path), Task.from_dir(path)) + + def __post_init__(self): + """Validate the data.""" + data = self.data + is_numpy = self._is_numpy() + + # >>> Check data types. + for key, allowed_dtypes in { + 'x_num': [np.dtype('float32')] if is_numpy else [torch.float32], + 'x_bin': [np.dtype('float32')] if is_numpy else [torch.float32], + 'x_cat': [] if is_numpy else [torch.int64], + 'y': ( + [np.dtype('float32'), np.dtype('float64'), np.dtype('int64')] + if is_numpy + else [torch.float32, torch.int64] + ), + }.items(): + if key in data: + for part, value in data[key].items(): + if key == 'x_cat' and is_numpy: + assert value.dtype in ( + np.dtype('int32'), + np.dtype('int64'), + ) or isinstance( + value.dtype, + np.dtypes.StrDType, # type: ignore[code] + ) + else: + assert value.dtype in allowed_dtypes, ( + f'The value data["{key}"]["{part}"] has dtype' + f' {value.dtype}, but it must be one of {allowed_dtypes}' + ) + + # >>> Fix data types. + if self.task.is_regression: + for key in data['y']: + data['y'][key] = ( + data['y'][key].astype('float32') + if self._is_numpy() + else data['y'][key].to(torch.float32) + ) + if 'x_cat' in data and data['x_cat']['train'].dtype == np.dtype('int32'): + for key in data['x_cat']: + data['x_cat'][key] = data['x_cat'][key].astype('int64') + + # >>> Check nans. + isnan = np.isnan if is_numpy else torch.isnan + for key in ['x_bin']: + if key in data: # type: ignore[code] + for part, value in data[key].items(): + assert not isnan( + value # type: ignore[code] + ).any(), f'data[\"{key}\"][\"{part}\"] contains nans' + for part, value in data['y'].items(): + assert not isnan(value).any(), f'data["y"]["{part}"] contains nans' # type: ignore[code] + + def _is_numpy(self) -> bool: + return isinstance(self.data['y']['train'], np.ndarray) + + def __contains__(self, key: DataKey) -> bool: + return key in self.data + + def __getitem__(self, key: DataKey) -> dict[PartKey, T]: + return self.data[key] + + def __setitem__(self, key: DataKey, value: dict[PartKey, T]) -> None: + self.data[key] = value + + @property + def n_num_features(self) -> int: + return self.data['x_num']['train'].shape[1] if 'x_num' in self.data else 0 + + @property + def n_bin_features(self) -> int: + return self.data['x_bin']['train'].shape[1] if 'x_bin' in self.data else 0 + + @property + def n_cat_features(self) -> int: + return self.data['x_cat']['train'].shape[1] if 'x_cat' in self.data else 0 + + @property + def n_features(self) -> int: + return self.n_num_features + self.n_bin_features + self.n_cat_features + + def size(self, part: None | PartKey) -> int: + return ( + sum(map(len, self.data['y'].values())) + if part is None + else len(self.data['y'][part]) + ) + + def parts(self) -> Iterable[PartKey]: + return self.data['y'].keys() + + def compute_cat_cardinalities(self) -> list[int]: + x_cat = self.data.get('x_cat') + if x_cat is None: + return [] + unique = np.unique if self._is_numpy() else torch.unique + return ( + [] + if x_cat is None + else [len(unique(column)) for column in x_cat['train'].T] + ) + + def to_torch(self, device: None | str | torch.device) -> 'Dataset[Tensor]': + return Dataset( + { + key: { + part: torch.as_tensor(value).to(device) + for part, value in self.data[key].items() + } + for key in self.data + }, + self.task, + ) + + +class NumPolicy(enum.Enum): + STANDARD = 'standard' + NOISY_QUANTILE = 'noisy-quantile' + + +def transform_num( + X_num: dict[PartKey, np.ndarray], policy: None | str | NumPolicy, seed: None | int +) -> dict[PartKey, np.ndarray]: + if policy is not None: + policy = NumPolicy(policy) + X_num_train = X_num['train'] + if policy == NumPolicy.STANDARD: + normalizer = sklearn.preprocessing.StandardScaler() + elif policy == NumPolicy.NOISY_QUANTILE: + normalizer = sklearn.preprocessing.QuantileTransformer( + n_quantiles=max(min(X_num['train'].shape[0] // 30, 1000), 10), + output_distribution='normal', + subsample=1_000_000_000, + random_state=seed, + ) + assert seed is not None + X_num_train = X_num_train + np.random.RandomState(seed).normal( + 0.0, 1e-5, X_num_train.shape + ).astype(X_num_train.dtype) + else: + raise ValueError(f'Unknown policy={policy}') + + normalizer.fit(X_num_train) + X_num = {k: normalizer.transform(v) for k, v in X_num.items()} # type: ignore[code] + + # NOTE + # (This is not a good way to process NaNs) + # This is a quick hack to stop failing on some datasets because of NaNs. + # NaNs are replaced with zeros (zero is the mean value for all features after + # the conventional preprocessing techniques). + X_num = {k: np.nan_to_num(v) for k, v in X_num.items()} + + # Remove columns with one constant value. + mask = np.array([len(np.unique(x)) > 1 for x in X_num['train'].T]) + X_num = {k: v[:, mask] for k, v in X_num.items()} + + X_num = {k: v.astype(np.float32) for k, v in X_num.items()} + return X_num + + +class CatPolicy(enum.Enum): + ORDINAL = 'ordinal' + ONE_HOT = 'one-hot' + + +def transform_cat( + X_cat: dict[PartKey, np.ndarray], policy: None | str | CatPolicy +) -> dict[PartKey, np.ndarray]: + if policy is None: + return X_cat + + policy = CatPolicy(policy) + + # The first step is always the ordinal encoding, + # even for the one-hot encoding. + unknown_value = np.iinfo('int64').max - 3 + encoder = sklearn.preprocessing.OrdinalEncoder( + handle_unknown='use_encoded_value', # type: ignore[code] + unknown_value=unknown_value, # type: ignore[code] + dtype='int64', # type: ignore[code] + ).fit(X_cat['train']) + X_cat = {k: encoder.transform(v) for k, v in X_cat.items()} + max_values = X_cat['train'].max(axis=0) + for part in ['val', 'test']: + part = cast(PartKey, part) + for column_idx in range(X_cat[part].shape[1]): + X_cat[part][X_cat[part][:, column_idx] == unknown_value, column_idx] = ( + max_values[column_idx] + 1 + ) + + if policy == CatPolicy.ORDINAL: + return X_cat + elif policy == CatPolicy.ONE_HOT: + encoder = sklearn.preprocessing.OneHotEncoder( + handle_unknown='ignore', + sparse=False, + dtype=np.float32, # type: ignore[code] + ) + encoder.fit(X_cat['train']) + return {k: cast(np.ndarray, encoder.transform(v)) for k, v in X_cat.items()} + else: + raise ValueError(f'Unknown policy={policy}') + + +@dataclass(frozen=True, kw_only=True) +class RegressionLabelStats: + mean: float + std: float + + +def standardize_labels( + y: dict[PartKey, np.ndarray], +) -> tuple[dict[PartKey, np.ndarray], RegressionLabelStats]: + assert y['train'].dtype == np.dtype('float32') + mean = float(y['train'].mean()) + std = float(y['train'].std()) + return {k: (v - mean) / std for k, v in y.items()}, RegressionLabelStats( + mean=mean, std=std + ) + + +def build_dataset( + path: str | Path, + *, + num_policy: None | str | NumPolicy = None, + cat_policy: None | str | CatPolicy = None, + seed: int = 0, + cache: bool = False, +) -> Dataset[np.ndarray]: + path = Path(path).resolve() + if cache: + args = locals() + args.pop('cache') + args.pop('path') + cache_path = env.get_cache_dir() / ( + f'build_dataset__{path.name}__{hashlib.md5(str(args).encode("utf-8")).hexdigest()}.pickle' + ) + if cache_path.exists(): + cached_args, cached_value = pickle.loads(cache_path.read_bytes()) + assert args == cached_args, f'Hash collision for {cache_path}' + logger.info(f'Using cached dataset: {cache_path.name}') + return cached_value + else: + args = None + cache_path = None + + dataset = Dataset.from_dir(path) + if 'x_num' in dataset.data: + dataset['x_num'] = transform_num(dataset['x_num'], num_policy, seed) + if 'x_cat' in dataset.data: + dataset['x_cat'] = transform_cat(dataset['x_cat'], cat_policy) + + if cache_path is not None: + cache_path.write_bytes(pickle.dumps((args, dataset))) + return dataset diff --git a/tabm_integrated/paper/lib/deep.py b/tabm_integrated/paper/lib/deep.py new file mode 100644 index 000000000..cf90f20b5 --- /dev/null +++ b/tabm_integrated/paper/lib/deep.py @@ -0,0 +1,651 @@ +import itertools +from typing import Any, Literal + +import rtdl_num_embeddings +import rtdl_revisiting_models +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch import Tensor +from torch.nn import Parameter + + +# ====================================================================================== +# Initialization +# ====================================================================================== +def init_rsqrt_uniform_(x: Tensor, d: int) -> Tensor: + assert d > 0 + d_rsqrt = d**-0.5 + return nn.init.uniform_(x, -d_rsqrt, d_rsqrt) + + +@torch.inference_mode() +def init_random_signs_(x: Tensor) -> Tensor: + return x.bernoulli_(0.5).mul_(2).add_(-1) + + +# ====================================================================================== +# Modules +# ====================================================================================== +class Identity(nn.Module): + def __init__(self, *args, **kwargs) -> None: + super().__init__() + + def forward(self, x: Tensor) -> Tensor: + return x + + +class NLinear(nn.Module): + """A stack of N linear layers. Each layer is applied to its own part of the input. + + **Shape** + + - Input: ``(B, N, in_features)`` + - Output: ``(B, N, out_features)`` + + The i-th linear layer is applied to the i-th matrix of the shape (B, in_features). + + Technically, this is a simplified version of delu.nn.NLinear: + https://yura52.github.io/delu/stable/api/generated/delu.nn.NLinear.html. + The difference is that this layer supports only 3D inputs + with exactly one batch dimension. By contrast, delu.nn.NLinear supports + any number of batch dimensions. + """ + + def __init__( + self, n: int, in_features: int, out_features: int, bias: bool = True + ) -> None: + super().__init__() + self.weight = Parameter(torch.empty(n, in_features, out_features)) + self.bias = Parameter(torch.empty(n, out_features)) if bias else None + self.reset_parameters() + + def reset_parameters(self): + d = self.weight.shape[-2] + init_rsqrt_uniform_(self.weight, d) + if self.bias is not None: + init_rsqrt_uniform_(self.bias, d) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + assert x.ndim == 3 + assert x.shape[-(self.weight.ndim - 1) :] == self.weight.shape[:-1] + + x = x.transpose(0, 1) + x = x @ self.weight + x = x.transpose(0, 1) + if self.bias is not None: + x = x + self.bias + return x + + +class PiecewiseLinearEmbeddings(rtdl_num_embeddings.PiecewiseLinearEmbeddings): + """ + This class simply adds the default values for `activation` and `version`. + """ + + def __init__( + self, + *args, + activation: bool = False, + version: None | Literal['A', 'B'] = 'B', + **kwargs, + ) -> None: + super().__init__(*args, **kwargs, activation=activation, version=version) + + +class OneHotEncoding0d(nn.Module): + # Input: (*, n_cat_features=len(cardinalities)) + # Output: (*, sum(cardinalities)) + + def __init__(self, cardinalities: list[int]) -> None: + super().__init__() + self._cardinalities = cardinalities + + def forward(self, x: Tensor) -> Tensor: + assert x.ndim >= 1 + assert x.shape[-1] == len(self._cardinalities) + + return torch.cat( + [ + # NOTE + # This is a quick hack to support out-of-vocabulary categories. + # + # Recall that lib.data.transform_cat encodes categorical features + # as follows: + # - In-vocabulary values receive indices from `range(cardinality)`. + # - All out-of-vocabulary values (i.e. new categories in validation + # and test data that are not presented in the training data) + # receive the index `cardinality`. + # + # As such, the line below will produce the standard one-hot encoding for + # known categories, and the all-zeros encoding for unknown categories. + # This may not be the best approach to deal with unknown values, + # but should be enough for our purposes. + F.one_hot(x[..., i], cardinality + 1)[..., :-1] + for i, cardinality in enumerate(self._cardinalities) + ], + -1, + ) + + +class ScaleEnsemble(nn.Module): + def __init__( + self, + k: int, + d: int, + *, + init: Literal['ones', 'normal', 'random-signs'], + ) -> None: + super().__init__() + self.weight = nn.Parameter(torch.empty(k, d)) + self._weight_init = init + self.reset_parameters() + + def reset_parameters(self) -> None: + if self._weight_init == 'ones': + nn.init.ones_(self.weight) + elif self._weight_init == 'normal': + nn.init.normal_(self.weight) + elif self._weight_init == 'random-signs': + init_random_signs_(self.weight) + else: + raise ValueError(f'Unknown weight_init: {self._weight_init}') + + def forward(self, x: Tensor) -> Tensor: + assert x.ndim >= 2 + return x * self.weight + + +class LinearEfficientEnsemble(nn.Module): + """ + This layer is a more configurable version of the "BatchEnsemble" layer + from the paper + "BatchEnsemble: An Alternative Approach to Efficient Ensemble and Lifelong Learning" + (link: https://arxiv.org/abs/2002.06715). + + First, this layer allows to select only some of the "ensembled" parts: + - the input scaling (r_i in the BatchEnsemble paper) + - the output scaling (s_i in the BatchEnsemble paper) + - the output bias (not mentioned in the BatchEnsemble paper, + but is presented in public implementations) + + Second, the initialization of the scaling weights is configurable + through the `scaling_init` argument. + + NOTE + The term "adapter" is used in the TabM paper only to tell the story. + The original BatchEnsemble paper does NOT use this term. So this class also + avoids the term "adapter". + """ + + r: None | Tensor + s: None | Tensor + bias: None | Tensor + + def __init__( + self, + in_features: int, + out_features: int, + bias: bool = True, + *, + k: int, + ensemble_scaling_in: bool, + ensemble_scaling_out: bool, + ensemble_bias: bool, + scaling_init: Literal['ones', 'random-signs'], + ): + assert k > 0 + if ensemble_bias: + assert bias + super().__init__() + + self.weight = nn.Parameter(torch.empty(out_features, in_features)) + self.register_parameter( + 'r', + ( + nn.Parameter(torch.empty(k, in_features)) + if ensemble_scaling_in + else None + ), # type: ignore[code] + ) + self.register_parameter( + 's', + ( + nn.Parameter(torch.empty(k, out_features)) + if ensemble_scaling_out + else None + ), # type: ignore[code] + ) + self.register_parameter( + 'bias', + ( + nn.Parameter(torch.empty(out_features)) # type: ignore[code] + if bias and not ensemble_bias + else nn.Parameter(torch.empty(k, out_features)) + if ensemble_bias + else None + ), + ) + + self.in_features = in_features + self.out_features = out_features + self.k = k + self.scaling_init = scaling_init + + self.reset_parameters() + + def reset_parameters(self): + init_rsqrt_uniform_(self.weight, self.in_features) + scaling_init_fn = {'ones': nn.init.ones_, 'random-signs': init_random_signs_}[ + self.scaling_init + ] + if self.r is not None: + scaling_init_fn(self.r) + if self.s is not None: + scaling_init_fn(self.s) + if self.bias is not None: + bias_init = torch.empty( + # NOTE: the shape of bias_init is (out_features,) not (k, out_features). + # It means that all biases have the same initialization. + # This is similar to having one shared bias plus + # k zero-initialized non-shared biases. + self.out_features, + dtype=self.weight.dtype, + device=self.weight.device, + ) + bias_init = init_rsqrt_uniform_(bias_init, self.in_features) + with torch.inference_mode(): + self.bias.copy_(bias_init) + + def forward(self, x: Tensor) -> Tensor: + # x.shape == (B, K, D) + assert x.ndim == 3 + + # >>> The equation (5) from the BatchEnsemble paper (arXiv v2). + if self.r is not None: + x = x * self.r + x = x @ self.weight.T + if self.s is not None: + x = x * self.s + # <<< + + if self.bias is not None: + x = x + self.bias + return x + + +class LinearEfficientEnsembleRankR(nn.Module): + """Rank-r generalization of `LinearEfficientEnsemble`. + + Replaces the per-member rank-1 outer-product adapter `s_i r_i^T` + with a rank-r factorization `S_i R_i^T`, where + `R_i in R^{d_in x rank}` and `S_i in R^{d_out x rank}`. + + The per-member weight is `W_i = W ⊙ (S_i R_i^T)`. Equivalently, + the rank-r forward is `r` parallel rank-1 paths sharing `W`, + summed before the bias is added: + + y_i = sum_{j=1..rank} s_{i,j} ⊙ (W (r_{i,j} ⊙ x)) + b_i + + At ``rank == 1`` and ``additive == False`` this reduces exactly to + `LinearEfficientEnsemble` with both ``ensemble_scaling_in`` and + ``ensemble_scaling_out`` set to True. + + The ``additive`` flag switches the construction to an *additive* + LoRA-style correction `W_i = W + S_i R_i^T` for the defensive ablation + in Section 6.3 of the RLA spec. In this mode the per-member weight + starts equal to ``W`` because R/S are zero-initialized. + """ + + bias: None | Tensor + + def __init__( + self, + in_features: int, + out_features: int, + bias: bool = True, + *, + k: int, + rank: int = 1, + scaling_init: Literal['ones', 'random-signs'] = 'ones', + additive: bool = False, + init_mode: Literal['variance_preserving', 'base_preserving'] = 'variance_preserving', + base_preserve_noise: float = 1e-3, + ) -> None: + """ + ``init_mode`` selects how the rank-r adapter is initialised: + + * ``variance_preserving`` (default, original behaviour): every + column of R and S is filled with ``1/sqrt(r)`` so the sum over + rank paths preserves the variance of the rank-1 baseline. + For first-layer ``random-signs`` scaling, each R column gets + independent random signs scaled by ``1/sqrt(r)``. The forward + output is *not* bit-identical to the rank-1 baseline at rank>1 + (the per-path random signs interact differently). + + * ``base_preserving``: rank path 0 is initialised exactly as the + rank-1 baseline (R[:,:,0]=1 or random-signs, S[:,:,0]=1). + Extra rank paths j>=1 are initialised with S[:,:,j]=0 and + R[:,:,j]=tiny noise (std=``base_preserve_noise``). At init the + extra paths contribute zero to the forward output, so a + base-preserving rank-r RLA layer produces *bit-identical* + forward outputs to the baseline (rank-1) layer. After one + optimiser step S becomes non-zero and R picks up gradient. + """ + assert k > 0 + assert rank >= 1 + super().__init__() + + self.in_features = in_features + self.out_features = out_features + self.k = k + self.rank = rank + self.scaling_init = scaling_init + self.additive = additive + self.init_mode = init_mode + self.base_preserve_noise = base_preserve_noise + + self.weight = nn.Parameter(torch.empty(out_features, in_features)) + # R: (k, d_in, rank), S: (k, d_out, rank) + self.R = nn.Parameter(torch.empty(k, in_features, rank)) + self.S = nn.Parameter(torch.empty(k, out_features, rank)) + self.register_parameter( + 'bias', + nn.Parameter(torch.empty(k, out_features)) if bias else None, + ) + + self.reset_parameters() + + def reset_parameters(self) -> None: + init_rsqrt_uniform_(self.weight, self.in_features) + + if self.additive: + # Additive LoRA-style start at the baseline weight: zero S, normal R. + nn.init.zeros_(self.S) + init_rsqrt_uniform_(self.R, self.in_features) + elif self.init_mode == 'base_preserving': + # Path 0 reproduces the rank-1 baseline exactly. Extra paths + # are zero-gated on the S side (so they contribute nothing to + # the forward output at init) and receive tiny noise on the + # R side (so the next step gives S non-zero gradient). + # + # NOTE on RNG ordering: to keep the shared weight and bias + # element-identical to the baseline class under the same seed, + # the only RNG-consuming step inside the adapter init must + # match what the baseline does (ones-fill = no RNG). The + # extra-path noise *would* break that match, so we defer it + # until after the bias has been drawn (see below). + with torch.inference_mode(): + # Path 0 of S: deterministic 1.0 (matches baseline 's=ones'). + self.S.zero_() + self.S[:, :, 0].fill_(1.0) + # Path 0 of R: depends on scaling_init. + self.R.zero_() + if self.scaling_init == 'random-signs': + # Note: this draws RNG, just like the baseline class + # does for its own r=random-signs init, so equivalent + # baseline + base_preserving construction stays in sync. + sign_path = torch.empty_like(self.R[:, :, 0]) + init_random_signs_(sign_path) + self.R[:, :, 0].copy_(sign_path) + elif self.scaling_init == 'ones': + self.R[:, :, 0].fill_(1.0) + else: + raise ValueError(f'Unknown scaling_init: {self.scaling_init}') + elif self.init_mode == 'variance_preserving': + # Variance-preserving (original) form. Match the rank-1 + # baseline exactly only at rank=1. + scale = self.rank**-0.5 + if self.scaling_init == 'random-signs': + init_random_signs_(self.R) + self.R.data.mul_(scale) + with torch.inference_mode(): + self.S.fill_(scale) + elif self.scaling_init == 'ones': + with torch.inference_mode(): + self.R.fill_(scale) + self.S.fill_(scale) + else: + raise ValueError(f'Unknown scaling_init: {self.scaling_init}') + else: + raise ValueError(f'Unknown init_mode: {self.init_mode}') + + if self.bias is not None: + bias_init = torch.empty( + self.out_features, + dtype=self.weight.dtype, + device=self.weight.device, + ) + bias_init = init_rsqrt_uniform_(bias_init, self.in_features) + with torch.inference_mode(): + self.bias.copy_(bias_init) + + # Deferred extra-path noise for base_preserving. + # + # CRITICAL: this draw must NOT advance the global RNG, otherwise + # downstream layers in the same model will be initialised at a + # different RNG state than the baseline class would have used, + # breaking full-model bit-equivalence at rank 1 / rank > 1. + # We isolate the noise draw by save+restore of the global RNG + # state and use a separate Generator seeded deterministically + # from a hash of the parameter shape (so the noise pattern is + # reproducible across runs without consuming the main RNG). + if ( + self.init_mode == 'base_preserving' + and not self.additive + and self.rank > 1 + and self.base_preserve_noise > 0 + ): + # Deterministic per-layer seed: stable cross-process hash of + # the shape tuple via hashlib (Python's built-in hash() is + # process-salted by PYTHONHASHSEED, so identical configs would + # otherwise initialise the extra-path noise differently across + # processes; we want bitwise reproducibility). + import hashlib + shape_key = ( + f'rla_base_preserve_noise|' + f'{self.in_features}|{self.out_features}|{self.k}|{self.rank}' + ).encode('utf-8') + seed = ( + int.from_bytes(hashlib.sha256(shape_key).digest()[:4], 'big') + & 0x7FFFFFFF + ) + gen = torch.Generator(device=self.R.device) + gen.manual_seed(seed) + noise = torch.empty_like(self.R[:, :, 1:]) + noise.normal_(mean=0.0, std=self.base_preserve_noise, generator=gen) + with torch.inference_mode(): + self.R[:, :, 1:].copy_(noise) + + def forward(self, x: Tensor) -> Tensor: + # x.shape == (B, K, D_in) + assert x.ndim == 3 + # Backbone projection (shared across rank paths and members). + Wx = x @ self.weight.T # (B, K, D_out) + + if self.additive: + # W_i x = W x + S_i (R_i^T x). Sum the rank-r correction. + # x: (B,K,Din) R: (K,Din,r) -> z: (B,K,r) per member + z = torch.einsum('bki,kir->bkr', x, self.R) + # S: (K, Dout, r) -> correction: (B,K,Dout) + corr = torch.einsum('bkr,kor->bko', z, self.S) + y = Wx + corr + else: + # Multiplicative: y = sum_j S[:,:,j] ⊙ (W (R[:,:,j] ⊙ x)). + # Loop over rank (rank ≤ 8, GEMM-bound; loop overhead negligible). + y = x.new_zeros(x.shape[0], x.shape[1], self.out_features) + for j in range(self.rank): + # x_scaled: (B,K,Din) — element-wise scale by R[:,:,j] (K,Din) + x_scaled = x * self.R[:, :, j] + proj = x_scaled @ self.weight.T # (B,K,Dout) + y = y + proj * self.S[:, :, j] # broadcast (K,Dout) -> (B,K,Dout) + + if self.bias is not None: + y = y + self.bias + return y + + +def make_efficient_ensemble(module: nn.Module, EnsembleLayer, **kwargs) -> None: + """Replace linear layers with efficient ensembles of linear layers. + + NOTE + In the paper, there are no experiments with networks with normalization layers. + Perhaps, their trainable weights (the affine transformations) also need + "ensemblification" as in the paper about "FiLM-Ensemble". + Additional experiments are required to make conclusions. + """ + for name, submodule in list(module.named_children()): + if isinstance(submodule, nn.Linear): + module.add_module( + name, + EnsembleLayer( + in_features=submodule.in_features, + out_features=submodule.out_features, + bias=submodule.bias is not None, + **kwargs, + ), + ) + else: + make_efficient_ensemble(submodule, EnsembleLayer, **kwargs) + + +class MLP(nn.Module): + def __init__( + self, + *, + d_in: None | int = None, + d_out: None | int = None, + n_blocks: int, + d_block: int, + dropout: float, + activation: str = 'ReLU', + ) -> None: + super().__init__() + + d_first = d_block if d_in is None else d_in + self.blocks = nn.ModuleList( + [ + nn.Sequential( + nn.Linear(d_first if i == 0 else d_block, d_block), + getattr(nn, activation)(), + nn.Dropout(dropout), + ) + for i in range(n_blocks) + ] + ) + self.output = None if d_out is None else nn.Linear(d_block, d_out) + + def forward(self, x: Tensor) -> Tensor: + for block in self.blocks: + x = block(x) + if self.output is not None: + x = self.output(x) + return x + + +_CUSTOM_MODULES = { + # https://docs.python.org/3/library/stdtypes.html#definition.__name__ + CustomModule.__name__: CustomModule + for CustomModule in [ + rtdl_num_embeddings.LinearEmbeddings, + rtdl_num_embeddings.LinearReLUEmbeddings, + rtdl_num_embeddings.PeriodicEmbeddings, + PiecewiseLinearEmbeddings, + MLP, + ] +} + + +def make_module(type: str, *args, **kwargs) -> nn.Module: + Module = getattr(nn, type, None) + if Module is None: + Module = _CUSTOM_MODULES[type] + return Module(*args, **kwargs) + + +def get_n_parameters(m: nn.Module): + return sum(x.numel() for x in m.parameters() if x.requires_grad) + + +@torch.inference_mode() +def compute_parameter_stats(module: nn.Module) -> dict[str, dict[str, float]]: + stats = {'norm': {}, 'gradnorm': {}, 'gradratio': {}} + for name, parameter in module.named_parameters(): + stats['norm'][name] = parameter.norm().item() + if parameter.grad is not None: + stats['gradnorm'][name] = parameter.grad.norm().item() + # Avoid computing statistics for zero-initialized parameters. + if (parameter.abs() > 1e-6).any(): + stats['gradratio'][name] = ( + (parameter.grad.abs() / parameter.abs().clamp_min_(1e-6)) + .mean() + .item() + ) + stats['norm']['model'] = ( + torch.cat([x.flatten() for x in module.parameters()]).norm().item() + ) + stats['gradnorm']['model'] = ( + torch.cat([x.grad.flatten() for x in module.parameters() if x.grad is not None]) + .norm() + .item() + ) + return stats + + +# ====================================================================================== +# Optimization +# ====================================================================================== +def default_zero_weight_decay_condition( + module_name: str, module: nn.Module, parameter_name: str, parameter: Parameter +): + from rtdl_num_embeddings import _Periodic + + del module_name, parameter + return parameter_name.endswith('bias') or isinstance( + module, + nn.BatchNorm1d + | nn.LayerNorm + | nn.InstanceNorm1d + | rtdl_revisiting_models.LinearEmbeddings + | rtdl_num_embeddings.LinearEmbeddings + | rtdl_num_embeddings.LinearReLUEmbeddings + | _Periodic, + ) + + +def make_parameter_groups( + module: nn.Module, + zero_weight_decay_condition=default_zero_weight_decay_condition, + custom_groups: None | list[dict[str, Any]] = None, +) -> list[dict[str, Any]]: + if custom_groups is None: + custom_groups = [] + custom_params = frozenset( + itertools.chain.from_iterable(group['params'] for group in custom_groups) + ) + assert len(custom_params) == sum( + len(group['params']) for group in custom_groups + ), 'Parameters in custom_groups must not intersect' + zero_wd_params = frozenset( + p + for mn, m in module.named_modules() + for pn, p in m.named_parameters() + if p not in custom_params and zero_weight_decay_condition(mn, m, pn, p) + ) + default_group = { + 'params': [ + p + for p in module.parameters() + if p not in custom_params and p not in zero_wd_params + ] + } + return [ + default_group, + {'params': list(zero_wd_params), 'weight_decay': 0.0}, + *custom_groups, + ] + + +def make_optimizer(type: str, **kwargs) -> torch.optim.Optimizer: + Optimizer = getattr(torch.optim, type) + return Optimizer(**kwargs) diff --git a/tabm_integrated/paper/lib/env.py b/tabm_integrated/paper/lib/env.py new file mode 100644 index 000000000..42550d4f4 --- /dev/null +++ b/tabm_integrated/paper/lib/env.py @@ -0,0 +1,28 @@ +from __future__ import annotations +# NOTE: this file must not import anything from lib. + +from pathlib import Path + +_PROJECT_DIR = Path.cwd() + +assert _PROJECT_DIR.joinpath( + 'pixi.toml' +).exists(), 'The script must be run from the `paper/` directory' + + +def get_project_dir() -> Path: + return _PROJECT_DIR + + +def get_cache_dir() -> Path: + path = get_project_dir() / 'cache' + path.mkdir(exist_ok=True) + return path + + +def get_data_dir() -> Path: + return get_project_dir() / 'data' + + +def get_exp_dir() -> Path: + return get_project_dir() / 'exp' diff --git a/tabm_integrated/paper/lib/metrics.py b/tabm_integrated/paper/lib/metrics.py new file mode 100644 index 000000000..5df03a12e --- /dev/null +++ b/tabm_integrated/paper/lib/metrics.py @@ -0,0 +1,66 @@ +from __future__ import annotations +from typing import Any, cast + +import numpy as np +import scipy.special +import sklearn.metrics + +from .util import PredictionType, TaskType + + +def _get_labels_and_probs( + prediction: np.ndarray, + task_type: TaskType, + prediction_type: PredictionType, +) -> tuple[np.ndarray, None | np.ndarray]: + """Obtain labels and probabilities from raw predictions.""" + assert task_type in (TaskType.BINCLASS, TaskType.MULTICLASS) + + if prediction_type == PredictionType.LABELS: + return prediction, None + elif prediction_type == PredictionType.PROBS: + probs = prediction + elif prediction_type == PredictionType.LOGITS: + probs = ( + scipy.special.expit(prediction) + if task_type == TaskType.BINCLASS + else scipy.special.softmax(prediction, axis=1) + ) + else: + raise ValueError(f'Unknown prediction type: {prediction_type}') + + assert probs is not None + labels = np.round(probs) if task_type == TaskType.BINCLASS else probs.argmax(axis=1) + return labels.astype(np.int64), probs + + +def calculate_metrics( + y_true: np.ndarray, + y_pred: np.ndarray, + task_type: str | TaskType, + prediction_type: str | PredictionType, +) -> dict[str, Any]: + task_type = TaskType(task_type) + prediction_type = PredictionType(prediction_type) + + if task_type == TaskType.REGRESSION: + assert prediction_type == PredictionType.LABELS + result = { + 'rmse': float(sklearn.metrics.mean_squared_error(y_true, y_pred) ** 0.5), + 'mae': float(sklearn.metrics.mean_absolute_error(y_true, y_pred)), + 'r2': float(sklearn.metrics.r2_score(y_true, y_pred)), + } + + else: + assert prediction_type is not None + labels, probs = _get_labels_and_probs(y_pred, task_type, prediction_type) + result = cast( + dict[str, Any], + sklearn.metrics.classification_report(y_true, labels, output_dict=True), + ) + if probs is not None: + result['cross-entropy'] = sklearn.metrics.log_loss(y_true, probs) + if task_type == TaskType.BINCLASS and probs is not None: + result['roc-auc'] = sklearn.metrics.roc_auc_score(y_true, probs) + + return result diff --git a/tabm_integrated/paper/lib/util.py b/tabm_integrated/paper/lib/util.py new file mode 100644 index 000000000..fbc8fc951 --- /dev/null +++ b/tabm_integrated/paper/lib/util.py @@ -0,0 +1,517 @@ +from __future__ import annotations +import argparse +import datetime +import enum +import importlib +import inspect +import json +import os +import shutil +import sys +import time +from collections.abc import Callable +from copy import deepcopy +from pathlib import Path +from pprint import pprint +from typing import Any, TypeVar, cast + +import numpy as np +import tomli +import tomli_w +from loguru import logger + +# NOTE +# This file must NOT import anything from lib except for `env`, +# because all other submodules are allowed to import `util`. +from . import env + +# The purpose of the following snippet is to optimize import times +# when slow-to-import modules are not needed. +_TORCH = None + + +def _torch(): + global _TORCH + if _TORCH is None: + import torch + + _TORCH = torch + return _TORCH + + +# ====================================================================================== +# Const +# ====================================================================================== +WORST_SCORE = -999999.0 + +# ====================================================================================== +# Types +# ====================================================================================== +KWArgs = dict[str, Any] +JSONDict = dict[str, Any] # Must be JSON-serializable. + +DataKey = str # 'x_num', 'x_bin', 'x_cat', 'y', ... +PartKey = str # 'train', 'val', 'test', ... + + +class TaskType(enum.Enum): + REGRESSION = 'regression' + BINCLASS = 'binclass' + MULTICLASS = 'multiclass' + + +class PredictionType(enum.Enum): + LABELS = 'labels' + PROBS = 'probs' + LOGITS = 'logits' + + +class Score(enum.Enum): + ACCURACY = 'accuracy' + CROSS_ENTROPY = 'cross-entropy' + MAE = 'mae' + R2 = 'r2' + RMSE = 'rmse' + ROC_AUC = 'roc-auc' + + +# ====================================================================================== +# Tools for the `main` function. +# +# The following utilities expect that the `main` function +# has one of the following signatures: +# +# 1. main(config, output = None, *, force: bool = False) -> None | JSONDict +# 2. main(config, output = None, *, force: bool = False, continue_: bool = False) -> None | JSONDict # noqa +# +# Notes: +# * `config` is a Python dictionary or a path to a config in the TOML format. +# * `output` is the output directory with all results of the run. +# If not provided, it it automatically inferred from the config path. +# * Setting `force=True` means removing the already existing output. +# * Setting `continue_=True` means continuing the execution from a checkpoint. +# * The return value is `report` -- a JSON-serializable Python dictionary +# with any information about the run. +# ====================================================================================== +T = TypeVar('T') + + +def check( + config, output: None | str | Path, *, config_type: type[T] = dict +) -> tuple[T, Path]: + """Load the config and infer the path to the output directory.""" + # >>> This is a snippet for the internal infrastructure, ignore it. + snapshot_dir = os.environ.get('SNAPSHOT_PATH') + if snapshot_dir and Path(snapshot_dir).joinpath('CHECKPOINTS_RESTORED').exists(): + assert inspect.stack()[1].frame.f_locals.get('continue_') + del snapshot_dir + # <<< + + # >>> Check paths. + if isinstance(config, str | Path): + # config is a path. + config = Path(config) + assert config.suffix == '.toml' + assert config.exists(), f'The config {config} does not exist' + if output is None: + # In this case, output is a directory located next to the config. + output = config.with_suffix('') + config = load_config(config) + else: + # config is already a dictionary. + assert ( + output is not None + ), 'If config is a dictionary, then the `output` directory must be provided.' + output = Path(output).resolve() + + # >>> Check the config. + if config_type is dict: + pass + elif ( + # If all conditions are True, config_type is assumed to be a TypedDict. + issubclass(config_type, dict) + and hasattr(config_type, '__required_keys__') + and hasattr(config_type, '__optional_keys__') + ): + # >>> Check the keys. + presented_keys = frozenset(config) + required_keys = config_type.__required_keys__ # type: ignore[code] + optional_keys = config_type.__optional_keys__ # type: ignore[code] + assert presented_keys >= required_keys, ( + 'The config is missing the following required keys:' + f' {", ".join(required_keys - presented_keys)}' + ) + assert set(config) <= (required_keys | optional_keys), ( + 'The config has unknown keys:' + f' {", ".join(presented_keys - required_keys - optional_keys)}' + ) + + return cast(T, config), output + + +def start(output: str | Path, *, continue_: bool = False, force: bool = False) -> bool: + """Create the output directory (if missing). + + Returns: + True if the caller should continue the execution. + False if the caller should immediately return. + """ + print_sep('=') + output = Path(output).resolve() + print(f'[>>>] {try_get_relative_path(output)} | {datetime.datetime.now()}') + print_sep('=') + + if output.exists(): + if force: + logger.warning('Removing the existing output') + time.sleep(2.0) # Keep the above message visible for some time. + shutil.rmtree(output) + output.mkdir() + return True + elif not continue_: + backup_output(output) + logger.warning('The output already exists!') + return False + elif output.joinpath('DONE').exists(): + backup_output(output) + logger.info('Already done!\n') + return False + else: + logger.info('Continuing with the existing output') + return True + else: + logger.info('Creating the output') + output.mkdir() + return True + + +def create_report(function, config) -> JSONDict: + return { + 'function': get_function_full_name(function), + 'gpus': get_gpu_names(), + 'config': jsonify(config), + } + + +def summarize(report: JSONDict) -> JSONDict: + """Summarize the key information from the report.""" + summary = {'function': report.get('function')} + + if 'best' in report: + # The gpus info is collected from the best report. + summary['best'] = summarize(report['best']) + elif 'gpus' in report: + summary['gpus'] = report['gpus'] + + for key in ['n_parameters', 'best_stage', 'best_epoch', 'tuning_time', 'trial_id']: + if key in report: + summary[key] = deepcopy(report[key]) + + metrics = report.get('metrics') + if metrics is not None and 'score' in next(iter(metrics.values())): + summary['scores'] = {part: metrics[part]['score'] for part in metrics} + + for key in ['n_completed_trials', 'time']: + if key in report: + summary[key] = deepcopy(report[key]) + + return summary + + +def finish(output: Path, report: JSONDict) -> None: + dump_report(output, report) + + # >>> A code block for the internal infrastructure, ignore it. + JSON_OUTPUT_FILE = os.environ.get('JSON_OUTPUT_FILE') + if JSON_OUTPUT_FILE: + try: + key = str(output.relative_to(env.get_project_dir())) + except ValueError: + pass + else: + json_output_path = Path(JSON_OUTPUT_FILE) + try: + json_data = json.loads(json_output_path.read_text()) + except (FileNotFoundError, json.decoder.JSONDecodeError): + json_data = {} + json_data[key] = load_report(output) + json_output_path.write_text(json.dumps(json_data, indent=4)) + shutil.copyfile( + json_output_path, + os.path.join(os.environ['SNAPSHOT_PATH'], 'json_output.json'), + ) + # <<< + + output.joinpath('DONE').touch() + backup_output(output) + print() + try: + print_summary(output) + except FileNotFoundError: + pass + print() + print_sep('=') + print(f'[<<<] {try_get_relative_path(output)} | {datetime.datetime.now()}') + print_sep('=') + + +def run(function: Callable[..., None | JSONDict]) -> None | JSONDict: + """Run CLI for the main function.""" + parser = argparse.ArgumentParser() + parser.add_argument('config') + parser.add_argument('--output') + parser.add_argument('--force', action='store_true') + if 'continue_' in inspect.signature(function).parameters: + parser.add_argument('--continue', action='store_true', dest='continue_') + + return function(**vars(parser.parse_args(sys.argv[1:]))) + + +# ====================================================================================== +# IO for the output directory +# ====================================================================================== +def load_config(output_or_config: str | Path) -> JSONDict: + return tomli.loads(Path(output_or_config).with_suffix('.toml').read_text()) + + +def dump_config( + output_or_config: str | Path, config: JSONDict, *, force: bool = False +) -> None: + config_path = Path(output_or_config).with_suffix('.toml') + if config_path.exists() and not force: + raise RuntimeError( + 'The following config already exists (pass force=True to overwrite it)' + f' {config_path}' + ) + config_path.write_text(tomli_w.dumps(config)) + + +def load_report(output: str | Path) -> JSONDict: + return json.loads(Path(output).joinpath('report.json').read_text()) + + +def dump_report(output: str | Path, report: JSONDict) -> None: + Path(output).joinpath('report.json').write_text(json.dumps(report, indent=4)) + + +def load_summary(output: str | Path) -> JSONDict: + return json.loads(Path(output).joinpath('summary.json').read_text()) + + +def print_summary(output: str | Path): + pprint(load_summary(output), sort_dicts=False, width=60) + + +def dump_summary(output: str | Path, summary: JSONDict) -> None: + Path(output).joinpath('summary.json').write_text(json.dumps(summary, indent=4)) + + +def load_predictions(output: str | Path) -> dict[PartKey, np.ndarray]: + x = np.load(Path(output) / 'predictions.npz') + return {key: x[key] for key in x} + + +def dump_predictions( + output: str | Path, predictions: dict[PartKey, np.ndarray] +) -> None: + np.savez(Path(output) / 'predictions.npz', **predictions) + + +def get_checkpoint_path(output: str | Path) -> Path: + return Path(output) / 'checkpoint.pt' + + +def load_checkpoint(output: str | Path, **kwargs) -> Any: + # PyTorch 2.6 changed torch.load default weights_only=True, which breaks + # loading our full training checkpoint objects unless explicitly disabled. + kwargs.setdefault('weights_only', False) + return _torch().load(get_checkpoint_path(output), **kwargs) + + +def dump_checkpoint(output: str | Path, checkpoint: JSONDict, **kwargs) -> None: + _torch().save(checkpoint, get_checkpoint_path(output), **kwargs) + + +# ====================================================================================== +# Printing +# ====================================================================================== +def print_sep(ch='-'): + print(ch * 100) + + +def print_config(config: dict) -> None: + print() + pprint(config, sort_dicts=False, width=100) + print() + + +def print_metrics(loss: float, metrics: dict) -> None: + print( + f'(val) {metrics["val"]["score"]:.3f}' + f' (test) {metrics["test"]["score"]:.3f}' + f' (loss) {loss:.5f}' + ) + + +def log_scores(metrics: dict) -> None: + logger.debug( + f'[val] {metrics["val"]["score"]:.4f} [test] {metrics["test"]["score"]:.4f}' + ) + + +# ====================================================================================== +# CUDA +# ====================================================================================== +def get_device(): # -> torch.device + torch = _torch() + return torch.device('cuda:0') if torch.cuda.is_available() else torch.device('cpu') + + +def is_dataparallel_available() -> bool: + torch = _torch() + return ( + torch.cuda.is_available() + and torch.cuda.device_count() > 1 + and 'CUDA_VISIBLE_DEVICES' in os.environ + ) + + +def get_gpu_names() -> list[str]: + return [ + _torch().cuda.get_device_name(i) for i in range(_torch().cuda.device_count()) + ] + + +def is_oom_exception(err: RuntimeError) -> bool: + return isinstance(err, _torch().cuda.OutOfMemoryError) or any( + x in str(err) + for x in [ + 'CUDA out of memory', + 'CUBLAS_STATUS_ALLOC_FAILED', + 'CUDA error: out of memory', + ] + ) + + +# ====================================================================================== +# Other +# ====================================================================================== +def configure_libraries(): + torch = _torch() + torch.set_num_threads(1) + torch.backends.cuda.matmul.allow_tf32 = False # type: ignore[code] + torch.backends.cudnn.allow_tf32 = False # type: ignore[code] + torch.backends.cudnn.benchmark = False # type: ignore[code] + torch.backends.cudnn.deterministic = True # type: ignore[code] + + logger.remove() + logger.add(sys.stderr, format='{message}') + + +def try_get_relative_path(path: str | Path) -> Path: + path = Path(path).resolve() + project_dir = env.get_project_dir() + return path.relative_to(project_dir) if project_dir in path.parents else path + + +def jsonify(value): + if value is None or isinstance(value, bool | int | float | str | bytes): + return value + elif isinstance(value, list): + return [jsonify(x) for x in value] + elif isinstance(value, dict): + return {k: jsonify(v) for k, v in value.items()} + else: + return '' + + +def are_valid_predictions(predictions: dict) -> bool: + # predictions: dict[PartKey, np.ndarray] + assert all(isinstance(x, np.ndarray) for x in predictions.values()) + return all(np.isfinite(x).all() for x in predictions.values()) + + +def import_(qualname: str) -> Any: + """ + Examples: + + >>> import_('bin.model.main') + """ + try: + module, name = qualname.rsplit('.', 1) + return getattr(importlib.import_module(module), name) + except Exception as err: + raise ValueError(f'Cannot import "{qualname}"') from err + + +def get_function_full_name(function: Callable) -> str: + """ + Examples: + + >>> # In the script bin/model.py + >>> get_function_full_name(main) == 'bin.model.main' + + >>> # In the script a/b/c/foo.py + >>> assert get_function_full_name(main) == 'a.b.c.foo.main' + """ + module = inspect.getmodule(function) + assert module is not None, 'Failed to locate the module of the function.' + + module_path = getattr(module, '__file__', None) + assert module_path is not None, ( + 'Failed to locate the module of the function.' + ' This can happen if the code is running in a Jupyter notebook.' + ) + + module_path = Path(module_path).resolve() + project_dir = env.get_project_dir() + assert project_dir in module_path.parents, ( + 'The module of the function must be located within the project directory: ' + f' {project_dir}' + ) + + module_full_name = str( + module_path.relative_to(project_dir).with_suffix('') + ).replace('/', '.') + return f'{module_full_name}.{function.__name__}' + + +_LAST_SNAPSHOT_TIME = None + + +def backup_output(output: Path) -> None: + """A function for the internal infrastructure, ignore it.""" + backup_dir = os.environ.get('TMP_OUTPUT_PATH') + snapshot_dir = os.environ.get('SNAPSHOT_PATH') + if backup_dir is None: + assert snapshot_dir is None + return + assert snapshot_dir is not None + + try: + relative_output_dir = output.relative_to(env.get_project_dir()) + except ValueError: + return + + for dir_ in [backup_dir, snapshot_dir]: + new_output = dir_ / relative_output_dir + prev_backup_output = new_output.with_name(new_output.name + '_prev') + new_output.parent.mkdir(exist_ok=True, parents=True) + if new_output.exists(): + new_output.rename(prev_backup_output) + shutil.copytree(output, new_output) + # The case for evaluate.py which automatically creates configs. + if output.with_suffix('.toml').exists(): + shutil.copyfile( + output.with_suffix('.toml'), new_output.with_suffix('.toml') + ) + if prev_backup_output.exists(): + shutil.rmtree(prev_backup_output) + + global _LAST_SNAPSHOT_TIME + if _LAST_SNAPSHOT_TIME is None or time.time() - _LAST_SNAPSHOT_TIME > 10 * 60: + import nirvana_dl.snapshot # type: ignore[code] + + nirvana_dl.snapshot.dump_snapshot() + _LAST_SNAPSHOT_TIME = time.time() + print('The snapshot was saved!') diff --git a/tabm_integrated/paper/pixi.toml b/tabm_integrated/paper/pixi.toml new file mode 100644 index 000000000..c6373bf4d --- /dev/null +++ b/tabm_integrated/paper/pixi.toml @@ -0,0 +1,58 @@ +[workspace] +channels = ["nvidia", "conda-forge", "pytorch", "nodefaults"] +name = "tabm" +platforms = ["linux-64", "osx-arm64", "osx-64"] + +[dependencies] +loguru = "==0.7.2" +numpy = "==1.26.3" +optuna = "==3.3.0" +pandas = "==2.1.1" +polars = "==1.5.0" +python = "==3.10.12" +pytorch = { version = "==2.0.1", channel = "pytorch" } +scikit-learn = "==1.3.1" +scipy = "==1.11.3" +tensorboard = "==2.15.2" +tomli = "==2.0.1" +tomli-w = "==1.0.0" +tqdm = "==4.66.1" +typing-extensions = "==4.8.0" +# Development (results do not depend on these packages) +bokeh = "==3.3.4" +holoviews = "==1.18.3" +hvplot = "==0.10.0" +ipython = "==8.16.1" +ipywidgets = "==8.1.3" +jupyterlab = "==4.1.1" +mypy = "==1.7.1" +panel = "==1.3.6" +pip = "==23.3.1" +psutil = "==5.9.5" +python-dotenv = "==1.0.1" +ruff = "==0.6.0" +seaborn = "==0.13.2" + +[pypi-dependencies] +delu = "==0.0.25" +rtdl_num_embeddings = "==0.0.11" +rtdl_revisiting_models = "==0.0.2" + +[pypi-options] +index-url = "https://pypi.org/simple" + +[feature.cuda] +system-requirements = { libc = { family = "glibc", version = "2.27" } } +# In fact, it should be this: +# system-requirements = { cuda = "11.8", libc = { family = "glibc", version = "2.27" } } +# But the "cuda" requirement breaks the project on macOS. + +# (As of pixi==0.29.0) The ".target.linux-64" part is a workaround for projects +# with multiple platforms AND platform-specific dependencies (e.g. cuda) AND PyPI dependencies. +# Details: https://github.com/prefix-dev/pixi/issues/1051#issuecomment-2088820974 +[feature.cuda.target.linux-64.dependencies] +pytorch-cuda = { version = "==11.8", channel = "pytorch" } + +[environments] +default = { solve-group = "default" } +cuda = { features = ["cuda"], solve-group = "default" } diff --git a/tabm_integrated/paper/pyproject.toml b/tabm_integrated/paper/pyproject.toml new file mode 100644 index 000000000..151c00d4d --- /dev/null +++ b/tabm_integrated/paper/pyproject.toml @@ -0,0 +1,13 @@ +[tool.mypy] +check_untyped_defs = true +ignore_missing_imports = true + +[tool.ruff] +target-version = "py310" +line-length = 88 + +[tool.ruff.lint] +extend-select = ["RUF", "UP", "E501", "E731"] + +[tool.ruff.format] +quote-style = "single" diff --git a/tabm_integrated/paper/tools/cf_fisd_teachers.py b/tabm_integrated/paper/tools/cf_fisd_teachers.py new file mode 100644 index 000000000..19480f8c7 --- /dev/null +++ b/tabm_integrated/paper/tools/cf_fisd_teachers.py @@ -0,0 +1,310 @@ +from __future__ import annotations + +import argparse +import json +import os +import sys +from pathlib import Path +from typing import Any + +if __name__ == '__main__': + _cwd = Path.cwd() + assert _cwd.joinpath( + 'pixi.toml' + ).exists(), 'The script must be run from the `paper/` directory' + sys.path.append(str(_cwd)) + del _cwd + +import delu +import numpy as np +import pandas as pd +try: + import tomllib +except ImportError: + import tomli as tomllib # type: ignore[no-redef] +from loguru import logger + +import lib +import lib.data + + +SUPPORTED_DATASETS = ( + 'sberbank-housing', + 'ecom-offers', + 'homesite-insurance', + 'cooking-time', + 'delivery-eta', +) +TEACHERS = ('xgb', 'lgbm', 'cat') + + +def _load_toml(path: Path) -> dict[str, Any]: + with path.open('rb') as f: + return tomllib.load(f) + + +def _build_combined_features(dataset) -> tuple[dict[str, pd.DataFrame], list[int]]: + parts = list(dataset.parts()) + n_num = dataset.n_num_features + n_bin = dataset.n_bin_features + n_cat = dataset.n_cat_features + + frames: dict[str, pd.DataFrame] = {part: pd.DataFrame() for part in parts} + for part in parts: + if 'x_num' in dataset: + frames[part] = pd.concat( + [frames[part], pd.DataFrame(dataset['x_num'][part])], axis=1 + ) + if 'x_cat' in dataset: + frames[part] = pd.concat( + [frames[part], pd.DataFrame(dataset['x_cat'][part].astype('int64'))], + axis=1, + ) + if 'x_bin' in dataset: + frames[part] = pd.concat( + [frames[part], pd.DataFrame(dataset['x_bin'][part].astype('int64'))], + axis=1, + ) + frames[part].columns = list(range(frames[part].shape[1])) + + cat_indices_in_combined: list[int] = [] + if 'x_cat' in dataset: + cat_indices_in_combined.extend(range(n_num, n_num + n_cat)) + if 'x_bin' in dataset: + cat_indices_in_combined.extend(range(n_num + n_cat, n_num + n_cat + n_bin)) + return frames, cat_indices_in_combined + + +def _train_xgb( + cfg: dict[str, Any], + X: dict[str, pd.DataFrame], + y: dict[str, np.ndarray], + cat_idx: list[int], + is_regression: bool, + is_binclass: bool, + seed: int, +) -> np.ndarray: + from xgboost import XGBClassifier, XGBRegressor + + df_X = {part: frame.astype(np.float32) for part, frame in X.items()} + + model_kwargs = dict(cfg['model']) + _force_cpu_in_model_cfg('xgb', model_kwargs) + extra = {'random_state': seed} + if is_regression: + model = XGBRegressor(**model_kwargs, **extra) + else: + eval_metric = 'auc' if is_binclass else 'merror' + model = XGBClassifier( + **model_kwargs, + **extra, + disable_default_eval_metric=True, + eval_metric=eval_metric, + ) + fit_kwargs = dict(cfg.get('fit', {})) + model.fit( + df_X['train'], + y['train'], + eval_set=[(df_X['val'], y['val'])], + **fit_kwargs, + ) + return np.asarray(model.feature_importances_, dtype=np.float64) + + +def _train_lgbm( + cfg: dict[str, Any], + X: dict[str, pd.DataFrame], + y: dict[str, np.ndarray], + cat_idx: list[int], + is_regression: bool, + is_binclass: bool, + seed: int, +) -> np.ndarray: + import lightgbm + from lightgbm import LGBMClassifier, LGBMRegressor + + model_kwargs = dict(cfg['model']) + _force_cpu_in_model_cfg('lgbm', model_kwargs) + stopping_rounds = model_kwargs.pop('stopping_rounds', None) + extra = {'random_state': seed} + if is_regression: + model = LGBMRegressor(**model_kwargs, **extra) + fit_extra = {'eval_metric': 'rmse'} + elif is_binclass: + model = LGBMClassifier(**model_kwargs, **extra) + fit_extra = {'eval_metric': 'auc'} + else: + model = LGBMClassifier(**model_kwargs, **extra) + fit_extra = {'eval_metric': 'multi_error'} + callbacks = [] + if stopping_rounds is not None: + callbacks.append(lightgbm.early_stopping(stopping_rounds=int(stopping_rounds))) + fit_kwargs = dict(cfg.get('fit', {})) + if cat_idx: + fit_extra['categorical_feature'] = list(cat_idx) + model.fit( + X['train'], + y['train'], + eval_set=[(X['val'], y['val'])], + callbacks=callbacks, + **fit_kwargs, + **fit_extra, + ) + importances = np.asarray(model.feature_importances_, dtype=np.float64) + if importances.sum() > 0.0: + return importances + return np.asarray( + model.booster_.feature_importance(importance_type='gain'), dtype=np.float64 + ) + + +def _train_cat( + cfg: dict[str, Any], + X: dict[str, pd.DataFrame], + y: dict[str, np.ndarray], + cat_idx: list[int], + is_regression: bool, + is_binclass: bool, + seed: int, +) -> np.ndarray: + from catboost import CatBoostClassifier, CatBoostRegressor, Pool + + model_kwargs = dict(cfg['model']) + _force_cpu_in_model_cfg('cat', model_kwargs) + model_kwargs.setdefault('random_seed', seed) + model_kwargs.setdefault('verbose', False) + model_kwargs.setdefault('allow_writing_files', False) + if is_regression: + model = CatBoostRegressor(**model_kwargs) + else: + model = CatBoostClassifier(**model_kwargs) + train_pool = Pool(X['train'], y['train'], cat_features=cat_idx if cat_idx else None) + val_pool = Pool(X['val'], y['val'], cat_features=cat_idx if cat_idx else None) + fit_kwargs = dict(cfg.get('fit', {})) + model.fit(train_pool, eval_set=val_pool, **fit_kwargs) + return np.asarray(model.get_feature_importance(type='PredictionValuesChange'), dtype=np.float64) + + +def _normalize_to_simplex(v: np.ndarray) -> np.ndarray: + v = np.clip(v.astype(np.float64), 0.0, None) + s = float(v.sum()) + if s > 0.0: + return (v / s).astype(np.float32) + return np.full_like(v, 1.0 / len(v), dtype=np.float32) + + +def _resolve_tuned_toml(tabred_root: Path, teacher: str, dataset: str) -> Path: + name_map = {'xgb': 'xgboost_', 'lgbm': 'lightgbm_', 'cat': 'catboost_'} + base = tabred_root / 'exp' / name_map[teacher] / dataset + candidates = [base / 'evaluation' / '0.toml', base / '0.toml'] + for p in candidates: + if p.exists(): + return p + return candidates[0] + + +def _force_cpu_in_model_cfg(teacher: str, model_cfg: dict[str, Any]) -> None: + if teacher == 'xgb': + model_cfg['device'] = 'cpu' + model_cfg['tree_method'] = model_cfg.get('tree_method', 'hist') + model_cfg.pop('gpu_id', None) + elif teacher == 'lgbm': + model_cfg['device_type'] = 'cpu' + model_cfg.pop('gpu_use_dp', None) + model_cfg.pop('gpu_platform_id', None) + model_cfg.pop('gpu_device_id', None) + elif teacher == 'cat': + model_cfg['task_type'] = 'CPU' + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument('--dataset', required=True, choices=SUPPORTED_DATASETS) + ap.add_argument('--seed', type=int, default=0) + ap.add_argument( + '--tabred-root', + type=Path, + default=Path(os.environ.get('TABRED_REPO_ROOT', 'D:/TabM_PROJ/tabred_fork')), + ) + ap.add_argument( + '--out-root', + type=Path, + default=Path('exp/cf_fisd/_teachers/tabred'), + ) + ap.add_argument('--teachers', nargs='+', default=list(TEACHERS), choices=list(TEACHERS)) + ap.add_argument('--data-path', default=None, + help='Override dataset path; defaults to data/.') + args = ap.parse_args() + + delu.random.seed(args.seed) + data_path = args.data_path or f'data/{args.dataset}' + dataset = lib.data.build_dataset(path=data_path, cache=True, cat_policy='ordinal') + if dataset.task.is_regression: + dataset.data['y'], _ = lib.data.standardize_labels(dataset.data['y']) + + X, cat_idx = _build_combined_features(dataset) + y = {part: np.asarray(dataset['y'][part]) for part in dataset.parts()} + + is_reg = dataset.task.is_regression + is_bin = bool(getattr(dataset.task, 'is_binclass', False)) + + out_dir = args.out_root / args.dataset + out_dir.mkdir(parents=True, exist_ok=True) + + n_num = dataset.n_num_features + n_cat = dataset.n_cat_features + n_bin = dataset.n_bin_features + n_features = n_num + n_cat + n_bin + logger.info( + f'{args.dataset}: n_num={n_num} n_cat={n_cat} n_bin={n_bin} n_features={n_features}' + ) + + trainers = {'xgb': _train_xgb, 'lgbm': _train_lgbm, 'cat': _train_cat} + for t in args.teachers: + toml_path = _resolve_tuned_toml(args.tabred_root, t, args.dataset) + if not toml_path.exists(): + raise FileNotFoundError(toml_path) + cfg = _load_toml(toml_path) + logger.info(f'training {t} for {args.dataset}: hp from {toml_path}') + importances = trainers[t]( + cfg=cfg, + X=X, + y=y, + cat_idx=cat_idx, + is_regression=is_reg, + is_binclass=is_bin, + seed=args.seed, + ) + if importances.shape != (n_features,): + raise ValueError( + f'importance shape {importances.shape} != ({n_features},) for {t}/{args.dataset}' + ) + importances = _normalize_to_simplex(importances) + out_path = out_dir / f'{t}.npy' + np.save(out_path, importances) + topk = np.argsort(importances)[::-1][:5].tolist() + entropy = float(-(importances * np.log(importances + 1e-12)).sum()) + logger.info( + f' saved {out_path} top5_idx={topk} entropy={entropy:.3f}' + ) + + meta = { + 'dataset': args.dataset, + 'task_type': dataset.task.type_.value, + 'n_num': n_num, + 'n_cat': n_cat, + 'n_bin': n_bin, + 'n_features': n_features, + 'cat_indices_in_combined': cat_idx, + 'feature_order': 'num,cat,bin', + 'seed': args.seed, + 'tabred_repo_root': str(args.tabred_root), + 'teachers': list(args.teachers), + } + (out_dir / 'meta.json').write_text(json.dumps(meta, indent=2)) + logger.info(f'wrote {out_dir / "meta.json"}') + + +if __name__ == '__main__': + lib.configure_libraries() + main() diff --git a/tabm_integrated/pbs/run_integrated_16gpu.pbs b/tabm_integrated/pbs/run_integrated_16gpu.pbs new file mode 100644 index 000000000..88373c19a --- /dev/null +++ b/tabm_integrated/pbs/run_integrated_16gpu.pbs @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -euo pipefail +cd "$PBS_O_WORKDIR" +N_GPUS=16 SEEDS="${SEEDS:-0}" ./tabm_integrated/tools/run_integrated_matrix.sh diff --git a/tabm_integrated/tools/aggregate_integrated_results.py b/tabm_integrated/tools/aggregate_integrated_results.py new file mode 100644 index 000000000..5bfbc330f --- /dev/null +++ b/tabm_integrated/tools/aggregate_integrated_results.py @@ -0,0 +1,409 @@ + +from __future__ import annotations + +import argparse +import csv +import json +import math +import statistics +import tomllib +from collections import defaultdict +from pathlib import Path +from typing import Any + +import tomli_w + +PAPER = Path(__file__).resolve().parents[1] / 'paper' +EXP_ROOT = PAPER / 'exp' / 'integrated' +SUMMARY_PATH = PAPER / 'exp' / 'final_integrated_summary.csv' +AUDIT_PATH = PAPER / 'exp' / 'final_integrated_audit.csv' +REPORT_PATH = PAPER.parent / 'FINAL_EXPERIMENT_REPORT.md' +SELECTED_PATH = PAPER / 'exp' / 'selected_integrated_configs.csv' +RESCUE_SELECTED_PATH = PAPER / 'exp' / 'rescue_selected_configs.csv' +FINAL_MANIFEST = EXP_ROOT / 'manifest_final.txt' + +DATASET_ORDER = ['sberbank-housing', 'ecom-offers', 'homesite-insurance', 'cooking-time', 'delivery-eta'] +FINAL_DISPLAY = ['baseline_plr', 'best_rla_only', 'best_esam_only', 'best_mfb_only', 'best_cf_fisd_only', 'best_combined'] + + +def infer_inference_mode(path: Path) -> str: + parent = path.parent.parent.name + if parent.endswith('-best-head-evaluation'): + return 'best-head' + if parent.endswith('-greedy-heads-evaluation'): + return 'greedy-heads' + return 'mean' + + +def mean_config_dir_for_result_dir(run_dir: Path) -> Path: + eval_dir = run_dir.parent + name = eval_dir.name + if name.endswith('-best-head-evaluation'): + mean_name = name.removesuffix('-best-head-evaluation') + '-evaluation' + elif name.endswith('-greedy-heads-evaluation'): + mean_name = name.removesuffix('-greedy-heads-evaluation') + '-evaluation' + else: + mean_name = name + return eval_dir.with_name(mean_name) / run_dir.name + + +def variant_from_result(path: Path) -> str: + name = path.parent.parent.name + for suffix in ['-best-head-evaluation', '-greedy-heads-evaluation', '-evaluation']: + if name.endswith(suffix): + return name.removesuffix(suffix) + return name + + +def dataset_from_result(path: Path) -> str: + return path.parent.parent.parent.name + + +def wave_from_result(path: Path) -> str: + return path.parent.parent.parent.parent.name + + +def score_key(report: dict[str, Any]) -> tuple[str, str, str]: + val = report.get('metrics', {}).get('val', {}) + if 'rmse' in val: + return 'RMSE', 'rmse', 'lower' + if 'roc-auc' in val: + return 'AUROC', 'roc-auc', 'higher' + if 'accuracy' in val: + return 'accuracy', 'accuracy', 'higher' + raise KeyError(f'Unknown metrics keys: {sorted(val)}') + + +def signed_delta(test_mean: float, baseline_mean: float, direction: str) -> tuple[float, float]: + if direction == 'lower': + delta = baseline_mean - test_mean + pct = 100.0 * delta / baseline_mean if baseline_mean else float('nan') + else: + delta = test_mean - baseline_mean + pct = 100.0 * delta / abs(baseline_mean) if baseline_mean else float('nan') + return delta, pct + + +def status_for(delta: float, baseline_std: float, n: int, invalid: bool) -> str: + if invalid or n != 3: + return 'invalid' + if abs(delta) <= 1e-12: + return 'tie' + if delta < 0: + return 'loss' + if abs(delta) > baseline_std: + return 'clear_win' + return 'weak_win' + + +def iter_reports(root: Path = EXP_ROOT): + for report_path in root.rglob('report.json'): + if not report_path.parent.joinpath('DONE').exists(): + continue + try: + report = json.loads(report_path.read_text()) + except Exception: + continue + mean_run_dir = mean_config_dir_for_result_dir(report_path.parent) + cfg_path = mean_run_dir.with_suffix('.toml') + yield report_path, cfg_path, report + + +def collect_rows() -> list[dict[str, Any]]: + rows = [] + for report_path, cfg_path, report in iter_reports(): + dataset = dataset_from_result(report_path) + variant = variant_from_result(report_path) + wave = wave_from_result(report_path) + inference = infer_inference_mode(report_path) + metric, key, direction = score_key(report) + rows.append({ + 'wave': wave, + 'dataset': dataset, + 'variant': variant, + 'seed': int(report.get('config', {}).get('seed', report_path.parent.name)), + 'metric': metric, + 'metric_key': key, + 'direction': direction, + 'validation_metric': float(report['metrics']['val'][key]), + 'validation_score': float(report['metrics']['val']['score']), + 'test_metric': float(report['metrics']['test'][key]), + 'inference_mode': inference, + 'config_path': str(cfg_path.relative_to(PAPER)) if cfg_path.exists() else str(cfg_path), + 'result_path': str(report_path.parent.relative_to(PAPER)), + 'failure': bool(report.get('failure')), + 'best_step': report.get('best_step'), + }) + return rows + + +def candidate_family(variant: str) -> str | None: + if variant == 'baseline_plr': + return 'baseline_plr' + if variant == 'rla_only' or variant.startswith('rla_rank'): + return 'best_rla_only' + if variant == 'esam_only' or variant.startswith('esam_rho'): + return 'best_esam_only' + if variant == 'mfb_only' or variant.startswith('mfb_keep'): + return 'best_mfb_only' + if variant == 'cf_fisd_only' or variant.startswith('cf_fisd_lambda'): + return 'best_cf_fisd_only' + if variant in {'all_four_combined','rla_esam','rla_mfb','rla_cf_fisd','esam_mfb','esam_cf_fisd','mfb_cf_fisd','all_minus_rla','all_minus_esam','all_minus_mfb','all_minus_cf_fisd'}: + return 'best_combined' + return None + + +def select_final_configs() -> None: + rows = [r for r in collect_rows() if r['wave'] in {'smoke', 'sweeps'} and r['seed'] == 0 and not r['failure']] + by = defaultdict(list) + for row in rows: + fam = candidate_family(row['variant']) + if fam and fam != 'baseline_plr': + by[(row['dataset'], fam)].append(row) + selected = [] + for dataset in DATASET_ORDER: + baseline_cfg = EXP_ROOT / 'smoke' / dataset / 'baseline_plr-evaluation' / '0.toml' + if not baseline_cfg.exists(): + baseline_cfg = EXP_ROOT / 'baseline_fidelity' / dataset / 'baseline_plr-evaluation' / '0.toml' + selected.append({'dataset': dataset, 'final_variant': 'baseline_plr', 'source_variant': 'baseline_plr', 'source_wave': 'smoke', 'inference_mode': 'mean', 'validation_metric': '', 'validation_score': '', 'source_config_path': str(baseline_cfg.relative_to(PAPER))}) + for fam in FINAL_DISPLAY[1:]: + candidates = by.get((dataset, fam), []) + if not candidates: + raise RuntimeError(f'No selection candidates for {dataset}/{fam}') + best = max(candidates, key=lambda r: r['validation_score']) + selected.append({'dataset': dataset, 'final_variant': fam, 'source_variant': best['variant'], 'source_wave': best['wave'], 'inference_mode': best['inference_mode'], 'validation_metric': best['validation_metric'], 'validation_score': best['validation_score'], 'source_config_path': best['config_path']}) + + SELECTED_PATH.parent.mkdir(parents=True, exist_ok=True) + with SELECTED_PATH.open('w', newline='') as f: + writer = csv.DictWriter(f, fieldnames=list(selected[0].keys()), lineterminator="\n") + writer.writeheader() + writer.writerows(selected) + + final_paths: list[Path] = [] + for row in selected: + source_cfg = PAPER / row['source_config_path'] + cfg = tomllib.loads(source_cfg.read_text()) + for seed in [0, 1, 2]: + cfg_seed = json.loads(json.dumps(cfg)) + cfg_seed['seed'] = seed + if cfg_seed.get('model', {}).get('mfb'): + cfg_seed['model']['mfb']['mask_seed'] = seed + out = EXP_ROOT / 'final' / row['dataset'] / f"{row['final_variant']}-evaluation" / f'{seed}.toml' + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(tomli_w.dumps(cfg_seed)) + final_paths.append(out) + FINAL_MANIFEST.write_text('\n'.join(str(p.relative_to(PAPER)) for p in final_paths) + '\n') + print(f'selected configs: {SELECTED_PATH.relative_to(PAPER)}') + print(f'final manifest: {FINAL_MANIFEST.relative_to(PAPER)} ({len(final_paths)} jobs)') + + +def task_type_from_metric(metric: str) -> str: + if metric == 'RMSE': + return 'regression' + if metric == 'AUROC': + return 'binclass' + return '' + + +def summarize_final() -> list[dict[str, Any]]: + selection = {} + if SELECTED_PATH.exists(): + with SELECTED_PATH.open() as f: + for row in csv.DictReader(f): + selection[(row['dataset'], row['final_variant'])] = row + rows = [r for r in collect_rows() if r['wave'] == 'final'] + wanted = set((d, v) for d in DATASET_ORDER for v in FINAL_DISPLAY) + grouped = defaultdict(list) + baseline_grouped = defaultdict(list) + for row in rows: + if row['variant'] == 'baseline_plr': + baseline_grouped[(row['dataset'], row['inference_mode'])].append(row) + if (row['dataset'], row['variant']) not in wanted: + continue + sel = selection.get((row['dataset'], row['variant'])) + if sel and row['inference_mode'] != sel['inference_mode']: + continue + if row['variant'] == 'baseline_plr' and row['inference_mode'] != 'mean': + continue + grouped[(row['dataset'], row['variant'])].append(row) + + out = [] + for dataset in DATASET_ORDER: + for variant in FINAL_DISPLAY: + rs = sorted(grouped.get((dataset, variant), []), key=lambda r: r['seed']) + n = len(rs) + invalid = n != 3 or any(r['failure'] for r in rs) + metric = rs[0]['metric'] if rs else '' + direction = rs[0]['direction'] if rs else '' + selected_inference = 'mean' if variant == 'baseline_plr' else selection.get((dataset, variant), {}).get('inference_mode', rs[0]['inference_mode'] if rs else '') + baseline_rs = sorted(baseline_grouped.get((dataset, selected_inference), []), key=lambda r: r['seed']) + baseline_invalid = len(baseline_rs) != 3 or any(r['failure'] for r in baseline_rs) + val_mean = statistics.mean([r['validation_metric'] for r in rs]) if rs else float('nan') + baseline_val_values = [r['validation_metric'] for r in baseline_rs] + base_val_mean = statistics.mean(baseline_val_values) if baseline_val_values else float('nan') + val_delta, val_pct = signed_delta(val_mean, base_val_mean, direction) if rs and baseline_rs else (float('nan'), float('nan')) + test_values = [r['test_metric'] for r in rs] + test_mean = statistics.mean(test_values) if test_values else float('nan') + test_std = statistics.stdev(test_values) if len(test_values) > 1 else 0.0 + baseline_values = [r['test_metric'] for r in baseline_rs] + base_mean = statistics.mean(baseline_values) if baseline_values else float('nan') + base_std = statistics.stdev(baseline_values) if len(baseline_values) > 1 else 0.0 + delta, pct = signed_delta(test_mean, base_mean, direction) if rs and baseline_rs else (float('nan'), float('nan')) + status = 'baseline' if variant == 'baseline_plr' else status_for(delta, base_std, n, invalid or baseline_invalid) + sel = selection.get((dataset, variant), {}) + source_variant = sel.get('source_variant', variant) + config_paths = ';'.join(r['config_path'] for r in rs) + result_paths = ';'.join(r['result_path'] for r in rs) + out.append({ + 'dataset': dataset, + 'task_type': task_type_from_metric(metric), + 'variant': variant, + 'source_variant': source_variant, + 'selected_config': source_variant, + 'metric': metric, + 'direction': direction, + 'metric_direction': direction, + 'validation_metric': val_mean, + 'matched_validation_baseline': base_val_mean, + 'matched_validation_delta': val_delta, + 'matched_validation_percent_delta': val_pct, + 'test_metric': test_mean, + 'mean': test_mean, + 'std': test_std, + 'test_mean': test_mean, + 'test_std': test_std, + 'n_seeds': n, + 'baseline_mean': base_mean, + 'baseline_std': base_std, + 'matched_baseline_inference_mode': selected_inference, + 'matched_baseline_mean': base_mean, + 'matched_baseline_std': base_std, + 'absolute_delta': delta, + 'percent_delta': pct, + 'precision': '3 seeds', + 'inference_mode': selected_inference, + 'selected_inference_mode': selected_inference, + 'config_path': config_paths, + 'result_path': result_paths, + 'matched_baseline_result_path': ';'.join(r['result_path'] for r in baseline_rs), + 'status': status, + }) + return out + +def write_csv(path: Path, rows: list[dict[str, Any]]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open('w', newline='') as f: + writer = csv.DictWriter(f, fieldnames=list(rows[0].keys()) if rows else [], lineterminator="\n") + writer.writeheader() + writer.writerows(rows) + + +def fmt(x: Any) -> str: + if isinstance(x, float): + if math.isnan(x): + return 'nan' + return f'{x:.6g}' + return str(x) + + +def write_report(summary_rows: list[dict[str, Any]]) -> None: + lines = ['# Integrated TabM final experiment report', ''] + lines += [ + 'Official TabM PLR/PiecewiseLinearEmbeddings baseline configs are preserved exactly; variants add only module flags for RLA, ESAM, MFB, and CF-FISD.', + 'Deltas and win/loss statuses are computed against `baseline_plr` using the same selected inference mode (`mean`, `best-head`, or `greedy-heads`) as the variant row.', + '', + ] + mean_baseline = { + r['dataset']: r + for r in summary_rows + if r['variant'] == 'baseline_plr' and r['inference_mode'] == 'mean' + } + changed = [] + for r in summary_rows: + if r['variant'] == 'baseline_plr': + continue + base = mean_baseline.get(r['dataset']) + if not base: + continue + old_delta, _ = signed_delta(float(r['mean']), float(base['mean']), r['direction']) + old_status = status_for(old_delta, float(base['std']), int(r['n_seeds']), int(r['n_seeds']) != 3) + if old_status != r['status']: + changed.append((r, old_status, old_delta)) + if changed: + lines += [ + '## Matched-inference correction', + '', + 'The earlier mean-baseline comparison changed when every variant was compared with the matching baseline inference mode. Rescue sweeps are selected against matched validation baselines only.', + '', + '| dataset | variant | inference | mean-baseline status | matched-baseline status | mean-baseline delta | matched delta |', + '|---|---|---|---|---|---:|---:|', + ] + for r, old_status, old_delta in changed: + lines.append( + f"| {r['dataset']} | {r['variant']} | {r['inference_mode']} | {old_status} | {r['status']} | {fmt(old_delta)} | {fmt(r['absolute_delta'])} |" + ) + lines += [''] + + lines += ['## Final 3-seed results', ''] + lines += [ + '| dataset | task | variant | selected config | metric | direction | inference | matched validation baseline | validation metric | matched baseline test mean ± std | result test mean ± std | delta | % delta | n | status | config path | result path |', + '|---|---|---|---|---|---|---|---:|---:|---:|---:|---:|---:|---:|---|---|---|', + ] + for r in summary_rows: + baseline_mean_std = f"{fmt(r['matched_baseline_mean'])} ± {fmt(r['matched_baseline_std'])}" + mean_std = f"{fmt(r['mean'])} ± {fmt(r['std'])}" + lines.append( + f"| {r['dataset']} | {r['task_type']} | {r['variant']} | {r['source_variant']} | {r['metric']} | {r['direction']} | {r['inference_mode']} | {fmt(r['matched_validation_baseline'])} | {fmt(r['validation_metric'])} | {baseline_mean_std} | {mean_std} | {fmt(r['absolute_delta'])} | {fmt(r['percent_delta'])} | {r['n_seeds']} | {r['status']} | `{r['config_path']}` | `{r['result_path']}` |" + ) + if RESCUE_SELECTED_PATH.exists(): + lines += ['', '## Rescue validation selections', ''] + lines += [ + '| dataset | final variant | selected rescue config | inference | validation metric | matched validation baseline | validation delta | validation status | confirmed |', + '|---|---|---|---|---:|---:|---:|---|---|', + ] + with RESCUE_SELECTED_PATH.open() as f: + for row in csv.DictReader(f): + lines.append( + f"| {row['dataset']} | {row['final_variant']} | {row['source_variant']} | {row['inference_mode']} | {row['validation_metric']} | {row['matched_validation_baseline']} | {row['validation_delta']} | {row['validation_status']} | {row['confirm_3seed']} |" + ) + lines += ['', '## Validation-selected configs', ''] + if SELECTED_PATH.exists(): + lines += ['| dataset | final variant | selected sweep variant | inference | validation metric |', '|---|---|---|---|---:|'] + with SELECTED_PATH.open() as f: + for row in csv.DictReader(f): + lines.append(f"| {row['dataset']} | {row['final_variant']} | {row['source_variant']} | {row['inference_mode']} | {row['validation_metric']} |") + lines += ['', '## Module wins vs matched baseline', ''] + for variant in FINAL_DISPLAY[1:]: + wins = [r['dataset'] for r in summary_rows if r['variant'] == variant and r['status'] in {'clear_win','weak_win'}] + lines.append(f"- `{variant}`: {', '.join(wins) if wins else 'none'}") + REPORT_PATH.write_text('\n'.join(lines) + '\n') + +def stage_wave(manifest: str) -> None: + rows = collect_rows() + print(f'aggregated complete reports: {len(rows)}') + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument('--stage', choices=['wave', 'select-final', 'final'], default='final') + parser.add_argument('--manifest') + args = parser.parse_args() + if args.stage == 'wave': + stage_wave(args.manifest or '') + return + if args.stage == 'select-final': + select_final_configs() + return + summary = summarize_final() + write_csv(SUMMARY_PATH, summary) + audit_rows = collect_rows() + if audit_rows: + write_csv(AUDIT_PATH, audit_rows) + write_report(summary) + print(f'summary: {SUMMARY_PATH.relative_to(PAPER)}') + print(f'audit: {AUDIT_PATH.relative_to(PAPER)}') + print(f'report: {REPORT_PATH.relative_to(PAPER.parent)}') + + +if __name__ == '__main__': + main() diff --git a/tabm_integrated/tools/delivery_rescue.py b/tabm_integrated/tools/delivery_rescue.py new file mode 100644 index 000000000..abf1d9531 --- /dev/null +++ b/tabm_integrated/tools/delivery_rescue.py @@ -0,0 +1,361 @@ +from __future__ import annotations + +import argparse +import csv +import itertools +import json +import math +import statistics +import sys +import tomllib +from collections import Counter, defaultdict +from copy import deepcopy +from pathlib import Path +from typing import Any + +sys.dont_write_bytecode = True + +import tomli_w + +TOOLS = Path(__file__).resolve().parent +PAPER = TOOLS.parent / 'paper' +EXP_ROOT = PAPER / 'exp' / 'integrated' +TARGETS_PATH = PAPER / 'exp' / 'rescue_targets.csv' +RESCUE_SELECTED_PATH = PAPER / 'exp' / 'rescue_selected_configs.csv' +SELECTED_PATH = PAPER / 'exp' / 'selected_integrated_configs.csv' +IND_MANIFEST = EXP_ROOT / 'manifest_delivery_rescue_individual.txt' +COMB_MANIFEST = EXP_ROOT / 'manifest_delivery_rescue_combined.txt' +FINAL_MANIFEST = EXP_ROOT / 'manifest_delivery_rescue_final.txt' +DATASET = 'delivery-eta' + +sys.path.insert(0, str(TOOLS)) +import generate_integrated_configs as gen # noqa: E402 +import aggregate_integrated_results as agg # noqa: E402 + +MODULE_VARIANTS = { + 'RLA': 'best_rla_only', + 'ESAM': 'best_esam_only', + 'MFB': 'best_mfb_only', + 'CF-FISD': 'best_cf_fisd_only', + 'combined': 'best_combined', +} + + +def ftag(x: float | int | bool) -> str: + if isinstance(x, bool): + return 't' if x else 'f' + if isinstance(x, int): + return str(x) + if x == 0: + return '0' + return f'{x:g}'.replace('-', 'm').replace('.', 'p') + + +def start_epoch(cfg: dict[str, Any], frac: float) -> int: + budget = int(cfg['n_epochs']) if int(cfg['n_epochs']) > 0 else 2 * int(cfg['patience']) + return int(math.ceil(budget * frac)) + + +def write_csv(path: Path, rows: list[dict[str, Any]]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + fields: list[str] = [] + for row in rows: + for key in row: + if key not in fields: + fields.append(key) + with path.open('w', newline='') as f: + writer = csv.DictWriter(f, fieldnames=fields, lineterminator='\n') + writer.writeheader() + writer.writerows(rows) + + +def read_csv(path: Path) -> list[dict[str, str]]: + with path.open(newline='') as f: + return list(csv.DictReader(f)) + + +def write_config(wave: str, variant: str, cfg: dict[str, Any], seed: int = 0) -> Path: + cfg = deepcopy(cfg) + cfg['seed'] = seed + if cfg.get('model', {}).get('mfb'): + cfg['model']['mfb']['mask_seed'] = seed + path = EXP_ROOT / wave / DATASET / f'{variant}-evaluation' / f'{seed}.toml' + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(tomli_w.dumps(cfg)) + return path + + +def apply_rla(cfg: dict[str, Any], rank: int, noise: float, lr: float, freeze: float) -> None: + gen.apply_rla(cfg, rank=rank, noise=noise) + cfg['model']['rla_first_only'] = False + cfg['rla_adapter_lr_multiplier'] = lr + cfg['rla_extra_paths_freeze_fraction'] = freeze + + +def apply_esam(cfg: dict[str, Any], rho: float, frac: float, adapter_only: bool) -> None: + gen.apply_esam(cfg, rho=rho) + cfg['esam_adapter_only'] = adapter_only + cfg['esam_start_epoch'] = start_epoch(cfg, frac) + + +def apply_mfb(cfg: dict[str, Any], keep: float, frac: float, group_mode: str) -> None: + gen.apply_mfb(cfg, keep=keep) + cfg['model']['mfb']['start_epoch'] = start_epoch(cfg, frac) + cfg['model']['mfb']['start_fraction'] = frac + cfg['model']['mfb']['group_mode'] = group_mode + cfg['model']['mfb']['categorical_handling'] = 'no_cat_drop' + + +def apply_cf(cfg: dict[str, Any], lam: float, mode: str) -> None: + gen.apply_cf_fisd(cfg, DATASET, lam=lam) + cf = cfg['cf_fisd'] + cf['start_epoch'] = start_epoch(cfg, 0.5) + cf['start_fraction'] = 0.5 + cf['mode'] = mode + if mode == 'softmax': + cf['variant'] = 'softmax' + elif mode == 'consensus_raw': + cf['variant'] = 'raw' + k = int(cfg['model']['k']) + cf['member_groups'] = {name: list(range(k)) for name in cf['teacher_names']} + else: + raise ValueError(mode) + + +def generate_targets() -> None: + rows = read_csv(PAPER / 'exp' / 'final_integrated_summary.csv') + out = [] + for row in rows: + if row['dataset'] != DATASET or row['variant'] == 'baseline_plr': + continue + module = {v: k for k, v in MODULE_VARIANTS.items()}[row['variant']] + out.append({ + 'priority': 0, + 'dataset': DATASET, + 'module': module, + 'final_variant': row['variant'], + 'current_source_variant': row['source_variant'], + 'current_inference_mode': row['inference_mode'], + 'metric': row['metric'], + 'direction': row['direction'], + 'matched_baseline_test_mean': row['matched_baseline_mean'], + 'current_test_mean': row['mean'], + 'current_absolute_delta': row['absolute_delta'], + 'current_percent_delta': row['percent_delta'], + 'current_status': row['status'], + 'rescue_reason': 'delivery_eta_priority' if row['status'] in {'clear_win', 'weak_win'} else 'matched_baseline_loss', + }) + order = {'RLA': 0, 'ESAM': 1, 'MFB': 2, 'CF-FISD': 3, 'combined': 4} + out.sort(key=lambda r: order[r['module']]) + write_csv(TARGETS_PATH, out) + print(f'{TARGETS_PATH.relative_to(PAPER)} rows={len(out)} counts={dict(Counter(r["module"] for r in out))}') + + +def generate_individual() -> None: + paths: list[Path] = [] + for rank, noise, lr, freeze in itertools.product([1, 2, 4], [0.0, 1e-5, 1e-4], [0.25, 0.5], [0.0, 0.5]): + cfg = gen.base_config(DATASET, 0) + apply_rla(cfg, rank, noise, lr, freeze) + paths.append(write_config('rescue_delivery_individual', f'deliv_rla_r{rank}_n{ftag(noise)}_lr{ftag(lr)}_fr{ftag(freeze)}', cfg)) + for rho, frac, adapter_only in itertools.product([0.00025, 0.0005, 0.001], [0.25, 0.5], [True, False]): + cfg = gen.base_config(DATASET, 0) + apply_esam(cfg, rho, frac, adapter_only) + paths.append(write_config('rescue_delivery_individual', f'deliv_esam_rho{ftag(rho)}_sf{ftag(frac)}_adapter{ftag(adapter_only)}', cfg)) + for keep, frac, group_mode in itertools.product([0.975, 0.99], [0.5, 0.75], ['numerical_only', 'per_member']): + cfg = gen.base_config(DATASET, 0) + apply_mfb(cfg, keep, frac, group_mode) + paths.append(write_config('rescue_delivery_individual', f'deliv_mfb_k{ftag(keep)}_sf{ftag(frac)}_{group_mode}', cfg)) + for lam, mode in itertools.product([0.005, 0.01, 0.02], ['consensus_raw', 'softmax']): + cfg = gen.base_config(DATASET, 0) + apply_cf(cfg, lam, mode) + paths.append(write_config('rescue_delivery_individual', f'deliv_cf_l{ftag(lam)}_{mode}', cfg)) + IND_MANIFEST.write_text('\n'.join(str(p.relative_to(PAPER)) for p in paths) + '\n') + print(f'{IND_MANIFEST.relative_to(PAPER)} configs={len(paths)}') + + +def family(variant: str) -> str | None: + if variant.startswith('deliv_rla_'): + return 'RLA' + if variant.startswith('deliv_esam_'): + return 'ESAM' + if variant.startswith('deliv_mfb_'): + return 'MFB' + if variant.startswith('deliv_cf_'): + return 'CF-FISD' + if variant.startswith('deliv_comb_'): + return 'combined' + return None + + +def delta(value: float, baseline: float, direction: str) -> tuple[float, float]: + if direction == 'lower': + d = baseline - value + return d, 100.0 * d / baseline if baseline else float('nan') + d = value - baseline + return d, 100.0 * d / abs(baseline) if baseline else float('nan') + + +def seed0_baselines(rows: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: + out = {} + for row in rows: + if row['wave'] == 'final' and row['dataset'] == DATASET and row['variant'] == 'baseline_plr' and row['seed'] == 0 and not row['failure']: + out[row['inference_mode']] = row + missing = [x for x in ['mean', 'best-head', 'greedy-heads'] if x not in out] + if missing: + raise RuntimeError(f'missing delivery seed-0 baselines: {missing}') + return out + + +def candidate_rows(wave: str) -> list[dict[str, Any]]: + rows = agg.collect_rows() + bases = seed0_baselines(rows) + out = [] + for row in rows: + if row['wave'] != wave or row['dataset'] != DATASET or row['seed'] != 0 or row['failure']: + continue + mod = family(row['variant']) + if mod is None: + continue + base = bases[row['inference_mode']] + d, pct = delta(row['validation_metric'], base['validation_metric'], row['direction']) + enriched = dict(row) + enriched['module'] = mod + enriched['final_variant'] = MODULE_VARIANTS[mod] + enriched['matched_validation_baseline'] = base['validation_metric'] + enriched['validation_delta'] = d + enriched['validation_percent_delta'] = pct + enriched['validation_status'] = 'validation_win' if d > 1e-12 else ('close_no_validation_win' if pct >= -0.1 else 'no_validation_win') + out.append(enriched) + return out + + +def best_by_module(wave: str, modules: list[str]) -> dict[str, dict[str, Any]]: + grouped: dict[str, list[dict[str, Any]]] = defaultdict(list) + for row in candidate_rows(wave): + grouped[row['module']].append(row) + best = {} + for mod in modules: + if not grouped.get(mod): + raise RuntimeError(f'no completed candidates for {mod} in {wave}') + best[mod] = max(grouped[mod], key=lambda r: (r['validation_delta'], r['validation_score'])) + return best + + +def copy_rla(dst: dict[str, Any], src: dict[str, Any]) -> None: + for key in ['rla_rank', 'rla_first_only', 'rla_additive', 'rla_init', 'rla_base_preserve_noise']: + if key in src['model']: + dst['model'][key] = src['model'][key] + for key in ['rla_adapter_lr_multiplier', 'rla_extra_paths_freeze_fraction']: + if key in src: + dst[key] = src[key] + + +def copy_esam(dst: dict[str, Any], src: dict[str, Any]) -> None: + for key in ['use_esam', 'esam_rho', 'esam_eps', 'esam_adapter_only', 'esam_memberwise', 'esam_warmup_epochs', 'esam_start_epoch', 'esam_end_epoch', 'esam_log_diagnostics', 'esam_diagnostics_every']: + if key in src: + dst[key] = src[key] + + +def copy_cf(dst: dict[str, Any], src: dict[str, Any]) -> None: + dst['cf_fisd'] = deepcopy(src['cf_fisd']) + + +def generate_combined() -> None: + best = best_by_module('rescue_delivery_individual', ['RLA', 'ESAM', 'MFB', 'CF-FISD']) + cfgs = {mod: tomllib.loads((PAPER / row['config_path']).read_text()) for mod, row in best.items()} + combos = { + 'deliv_comb_rla_esam': ('RLA', 'ESAM'), + 'deliv_comb_rla_cf': ('RLA', 'CF-FISD'), + 'deliv_comb_esam_cf': ('ESAM', 'CF-FISD'), + 'deliv_comb_rla_esam_cf': ('RLA', 'ESAM', 'CF-FISD'), + } + paths = [] + for variant, mods in combos.items(): + cfg = gen.base_config(DATASET, 0) + if 'RLA' in mods: + copy_rla(cfg, cfgs['RLA']) + if 'ESAM' in mods: + copy_esam(cfg, cfgs['ESAM']) + if 'CF-FISD' in mods: + copy_cf(cfg, cfgs['CF-FISD']) + paths.append(write_config('rescue_delivery_combined', variant, cfg)) + COMB_MANIFEST.write_text('\n'.join(str(p.relative_to(PAPER)) for p in paths) + '\n') + print(f'{COMB_MANIFEST.relative_to(PAPER)} configs={len(paths)}') + + +def select_and_make_final() -> None: + selected = best_by_module('rescue_delivery_individual', ['RLA', 'ESAM', 'MFB', 'CF-FISD']) + selected.update(best_by_module('rescue_delivery_combined', ['combined'])) + rows = [] + for mod in ['RLA', 'ESAM', 'MFB', 'CF-FISD', 'combined']: + row = selected[mod] + rows.append({ + 'dataset': DATASET, + 'module': mod, + 'final_variant': MODULE_VARIANTS[mod], + 'source_variant': row['variant'], + 'source_wave': row['wave'], + 'inference_mode': row['inference_mode'], + 'validation_metric': row['validation_metric'], + 'matched_validation_baseline': row['matched_validation_baseline'], + 'validation_delta': row['validation_delta'], + 'validation_percent_delta': row['validation_percent_delta'], + 'validation_status': row['validation_status'], + 'validation_score': row['validation_score'], + 'source_config_path': row['config_path'], + 'seed0_result_path': row['result_path'], + 'confirm_3seed': True, + }) + write_csv(RESCUE_SELECTED_PATH, rows) + + selection = read_csv(SELECTED_PATH) + repl = {(r['dataset'], r['final_variant']): r for r in rows} + updated = [] + for row in selection: + key = (row['dataset'], row['final_variant']) + if key in repl: + r = repl[key] + new = dict(row) + new.update({ + 'source_variant': r['source_variant'], + 'source_wave': r['source_wave'], + 'inference_mode': r['inference_mode'], + 'validation_metric': str(r['validation_metric']), + 'validation_score': str(r['validation_score']), + 'matched_validation_baseline': str(r['matched_validation_baseline']), + 'validation_delta': str(r['validation_delta']), + 'validation_percent_delta': str(r['validation_percent_delta']), + 'validation_status': r['validation_status'], + 'source_config_path': r['source_config_path'], + }) + updated.append(new) + else: + updated.append(row) + write_csv(SELECTED_PATH, updated) + + final_paths = [] + for r in rows: + cfg = tomllib.loads((PAPER / r['source_config_path']).read_text()) + for seed in [0, 1, 2]: + final_paths.append(write_config('final', r['final_variant'], cfg, seed)) + FINAL_MANIFEST.write_text('\n'.join(str(p.relative_to(PAPER)) for p in final_paths) + '\n') + print(f'{RESCUE_SELECTED_PATH.relative_to(PAPER)} rows={len(rows)}') + print(f'{FINAL_MANIFEST.relative_to(PAPER)} configs={len(final_paths)}') + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument('stage', choices=['targets', 'generate-individual', 'generate-combined', 'select-final']) + args = parser.parse_args() + if args.stage == 'targets': + generate_targets() + elif args.stage == 'generate-individual': + generate_individual() + elif args.stage == 'generate-combined': + generate_combined() + elif args.stage == 'select-final': + select_and_make_final() + + +if __name__ == '__main__': + main() diff --git a/tabm_integrated/tools/generate_integrated_configs.py b/tabm_integrated/tools/generate_integrated_configs.py new file mode 100644 index 000000000..396b06fc7 --- /dev/null +++ b/tabm_integrated/tools/generate_integrated_configs.py @@ -0,0 +1,294 @@ + +from __future__ import annotations + +import argparse +import copy +import shutil +from pathlib import Path +from typing import Any + +import tomli_w + +PAPER = Path(__file__).resolve().parents[1] / 'paper' +EXP_ROOT = PAPER / 'exp' / 'integrated' +TEACHER_ROOT = Path('../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred') + +DATASET_ORDER = [ + 'sberbank-housing', + 'ecom-offers', + 'homesite-insurance', + 'cooking-time', + 'delivery-eta', +] + +OFFICIAL_BASE: dict[str, dict[str, Any]] = { + 'sberbank-housing': { + 'batch_size': 256, + 'patience': 16, + 'n_epochs': -1, + 'gradient_clipping_norm': 1.0, + 'amp': True, + 'data': {'cache': True, 'path': 'data/sberbank-housing', 'num_policy': 'noisy-quantile', 'cat_policy': 'ordinal'}, + 'optimizer': {'lr': 0.0008922700423431547, 'weight_decay': 0.00021333759467820313}, + 'model': { + 'arch_type': 'tabm', 'k': 32, 'share_training_batches': False, + 'backbone': {'n_blocks': 3, 'd_block': 256, 'dropout': 0.3128700072370906}, + 'num_embeddings': {'type': 'PiecewiseLinearEmbeddings', 'd_embedding': 28}, + }, + 'bins': {'n_bins': 84}, + }, + 'ecom-offers': { + 'batch_size': 1024, + 'patience': 16, + 'n_epochs': -1, + 'gradient_clipping_norm': 1.0, + 'amp': True, + 'data': {'cache': True, 'path': 'data/ecom-offers', 'num_policy': 'noisy-quantile'}, + 'optimizer': {'lr': 0.00024262819114537424, 'weight_decay': 0.0001501852317298042}, + 'model': { + 'arch_type': 'tabm', 'k': 32, 'share_training_batches': False, + 'backbone': {'n_blocks': 1, 'd_block': 960, 'dropout': 0.0}, + 'num_embeddings': {'type': 'PiecewiseLinearEmbeddings', 'd_embedding': 16}, + }, + 'bins': {'n_bins': 47}, + }, + 'homesite-insurance': { + 'batch_size': 1024, + 'patience': 16, + 'n_epochs': -1, + 'gradient_clipping_norm': 1.0, + 'amp': True, + 'data': {'cache': True, 'path': 'data/homesite-insurance', 'num_policy': 'noisy-quantile', 'cat_policy': 'ordinal'}, + 'optimizer': {'lr': 0.0018580623030886075, 'weight_decay': 0.0001614529849348179}, + 'model': { + 'arch_type': 'tabm', 'k': 32, 'share_training_batches': False, + 'backbone': {'n_blocks': 3, 'd_block': 704, 'dropout': 0.0}, + 'num_embeddings': {'type': 'PiecewiseLinearEmbeddings', 'd_embedding': 8}, + }, + 'bins': {'n_bins': 15}, + }, + 'cooking-time': { + 'batch_size': 1024, + 'patience': 16, + 'n_epochs': -1, + 'gradient_clipping_norm': 1.0, + 'amp': True, + 'data': {'cache': True, 'path': 'data/cooking-time', 'cat_policy': 'ordinal'}, + 'optimizer': {'lr': 0.00012065020494450812, 'weight_decay': 0.07000081679295954}, + 'model': { + 'arch_type': 'tabm', 'k': 32, 'share_training_batches': False, + 'backbone': {'n_blocks': 1, 'd_block': 416, 'dropout': 0.0}, + 'num_embeddings': {'type': 'PiecewiseLinearEmbeddings', 'd_embedding': 16}, + }, + 'bins': {'n_bins': 8}, + }, + 'delivery-eta': { + 'batch_size': 1024, + 'patience': 16, + 'n_epochs': -1, + 'gradient_clipping_norm': 1.0, + 'amp': True, + 'data': {'cache': True, 'path': 'data/delivery-eta', 'cat_policy': 'ordinal'}, + 'optimizer': {'lr': 0.0025639267063470926, 'weight_decay': 0.0507074044872309}, + 'model': { + 'arch_type': 'tabm', 'k': 32, 'share_training_batches': False, + 'backbone': {'n_blocks': 2, 'd_block': 752, 'dropout': 0.18437573041970334}, + 'num_embeddings': {'type': 'PiecewiseLinearEmbeddings', 'd_embedding': 32}, + }, + 'bins': {'n_bins': 81}, + }, +} + +MODULE_DEFAULTS = { + 'rla': {'rank': 4, 'noise': 1e-4, 'inference': 'mean'}, + 'esam': {'rho': 0.005}, + 'mfb': {'keep': 0.90}, + 'cf_fisd': {'lambda': 0.10}, +} + + +def base_config(dataset: str, seed: int) -> dict[str, Any]: + cfg = copy.deepcopy(OFFICIAL_BASE[dataset]) + cfg['seed'] = int(seed) + cfg['head_selection'] = True + return cfg + + +def apply_rla(cfg: dict[str, Any], *, rank: int = 4, noise: float = 1e-4) -> None: + cfg['model']['rla_rank'] = int(rank) + cfg['model']['rla_first_only'] = False + cfg['model']['rla_additive'] = False + cfg['model']['rla_init'] = 'base_preserving' + cfg['model']['rla_base_preserve_noise'] = float(noise) + cfg['rla_adapter_lr_multiplier'] = 1.0 + cfg['rla_extra_paths_freeze_fraction'] = 0.0 + + +def apply_esam(cfg: dict[str, Any], *, rho: float = 0.005) -> None: + cfg['use_esam'] = True + cfg['esam_rho'] = float(rho) + cfg['esam_eps'] = 1e-12 + cfg['esam_adapter_only'] = True + cfg['esam_memberwise'] = True + cfg['esam_warmup_epochs'] = 0 + cfg['esam_start_epoch'] = 0 + cfg['esam_end_epoch'] = -1 + cfg['esam_log_diagnostics'] = False + cfg['esam_diagnostics_every'] = 100 + + +def apply_mfb(cfg: dict[str, Any], *, keep: float = 0.90) -> None: + cfg['model']['mfb'] = { + 'enabled': True, + 'mask_mode': 'member_fixed', + 'mask_granularity': 'feature_group', + 'keep_rate': float(keep), + 'training_only': False, + 'inverted_scaling': True, + 'use_soft_mask': False, + 'mask_strength': 1.0, + 'anchor_fraction': 0.0, + 'warmup_epochs': 0, + 'mask_seed': int(cfg['seed']), + } + + +def apply_cf_fisd(cfg: dict[str, Any], dataset: str, *, lam: float = 0.10) -> None: + cfg['cf_fisd'] = { + 'lambda': float(lam), + 'variant': 'raw', + 'dataset_name': dataset, + 'teacher_dir': str(TEACHER_ROOT / dataset), + 'teacher_names': ['xgb', 'lgbm', 'cat'], + } + + +def make_variant_config(dataset: str, seed: int, variant: str) -> dict[str, Any]: + cfg = base_config(dataset, seed) + if variant == 'baseline_plr': + return cfg + if variant.startswith('rla_rank'): + parts = variant.split('_') + apply_rla(cfg, rank=int(parts[1].removeprefix('rank')), noise=float(parts[2].removeprefix('noise'))) + return cfg + if variant.startswith('esam_rho'): + apply_esam(cfg, rho=float(variant.removeprefix('esam_rho'))) + return cfg + if variant.startswith('mfb_keep'): + apply_mfb(cfg, keep=float(variant.removeprefix('mfb_keep'))) + return cfg + if variant.startswith('cf_fisd_lambda'): + apply_cf_fisd(cfg, dataset, lam=float(variant.removeprefix('cf_fisd_lambda'))) + return cfg + + modules: set[str] + if variant == 'rla_only': + modules = {'rla'} + elif variant == 'esam_only': + modules = {'esam'} + elif variant == 'mfb_only': + modules = {'mfb'} + elif variant == 'cf_fisd_only': + modules = {'cf_fisd'} + elif variant == 'all_four_combined': + modules = {'rla', 'esam', 'mfb', 'cf_fisd'} + elif variant == 'rla_esam': + modules = {'rla', 'esam'} + elif variant == 'rla_mfb': + modules = {'rla', 'mfb'} + elif variant == 'rla_cf_fisd': + modules = {'rla', 'cf_fisd'} + elif variant == 'esam_mfb': + modules = {'esam', 'mfb'} + elif variant == 'esam_cf_fisd': + modules = {'esam', 'cf_fisd'} + elif variant == 'mfb_cf_fisd': + modules = {'mfb', 'cf_fisd'} + elif variant == 'all_minus_rla': + modules = {'esam', 'mfb', 'cf_fisd'} + elif variant == 'all_minus_esam': + modules = {'rla', 'mfb', 'cf_fisd'} + elif variant == 'all_minus_mfb': + modules = {'rla', 'esam', 'cf_fisd'} + elif variant == 'all_minus_cf_fisd': + modules = {'rla', 'esam', 'mfb'} + else: + raise KeyError(variant) + if 'rla' in modules: + apply_rla(cfg, rank=MODULE_DEFAULTS['rla']['rank'], noise=MODULE_DEFAULTS['rla']['noise']) + if 'esam' in modules: + apply_esam(cfg, rho=MODULE_DEFAULTS['esam']['rho']) + if 'mfb' in modules: + apply_mfb(cfg, keep=MODULE_DEFAULTS['mfb']['keep']) + if 'cf_fisd' in modules: + apply_cf_fisd(cfg, dataset, lam=MODULE_DEFAULTS['cf_fisd']['lambda']) + return cfg + + +def write_config(wave: str, dataset: str, variant: str, seed: int, *, force: bool) -> Path: + cfg = make_variant_config(dataset, seed, variant) + path = EXP_ROOT / wave / dataset / f'{variant}-evaluation' / f'{seed}.toml' + path.parent.mkdir(parents=True, exist_ok=True) + if force or not path.exists(): + path.write_text(tomli_w.dumps(cfg)) + return path + + +def write_manifest(name: str, paths: list[Path]) -> Path: + manifest = EXP_ROOT / f'manifest_{name}.txt' + manifest.parent.mkdir(parents=True, exist_ok=True) + manifest.write_text('\n'.join(str(p.relative_to(PAPER)) for p in paths) + ('\n' if paths else '')) + return manifest + + +def configs_for_stage(stage: str) -> dict[str, list[Path]]: + out: dict[str, list[Path]] = {} + if stage in {'initial', 'baseline_fidelity'}: + out['baseline_fidelity'] = [write_config('baseline_fidelity', d, 'baseline_plr', 0, force=True) for d in DATASET_ORDER] + if stage in {'initial', 'smoke'}: + variants = ['baseline_plr', 'rla_only', 'esam_only', 'mfb_only', 'cf_fisd_only', 'all_four_combined'] + out['smoke'] = [write_config('smoke', d, v, 0, force=True) for d in DATASET_ORDER for v in variants] + if stage in {'initial', 'sweeps'}: + variants: list[str] = [] + for rank in [2, 4, 8]: + for noise in [1e-3, 1e-4, 1e-5]: + variants.append(f'rla_rank{rank}_noise{noise:g}') + variants += [f'esam_rho{rho:g}' for rho in [0.001, 0.0025, 0.005, 0.01]] + variants += [f'mfb_keep{keep:g}' for keep in [0.70, 0.80, 0.90, 0.95]] + variants += [f'cf_fisd_lambda{lam:g}' for lam in [0.05, 0.10, 0.20]] + variants += ['rla_esam', 'rla_mfb', 'rla_cf_fisd', 'esam_mfb', 'esam_cf_fisd', 'mfb_cf_fisd', 'all_four_combined', 'all_minus_rla', 'all_minus_esam', 'all_minus_mfb', 'all_minus_cf_fisd'] + out['sweeps'] = [write_config('sweeps', d, v, 0, force=True) for d in DATASET_ORDER for v in variants] + return out + + +def verify_baseline_configs(paths: list[Path]) -> None: + import tomllib + for path in paths: + dataset = path.parts[-3] + cfg = tomllib.loads(path.read_text()) + expected = copy.deepcopy(OFFICIAL_BASE[dataset]) + for key, value in expected.items(): + if cfg.get(key) != value: + raise AssertionError(f'{path}: {key} differs from official template') + if cfg['seed'] != 0 or cfg.get('head_selection') is not True: + raise AssertionError(f'{path}: seed/head_selection wrapper invalid') + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument('--stage', choices=['initial', 'baseline_fidelity', 'smoke', 'sweeps'], default='initial') + parser.add_argument('--clean', action='store_true') + args = parser.parse_args() + if args.clean and EXP_ROOT.exists(): + shutil.rmtree(EXP_ROOT) + generated = configs_for_stage(args.stage) + for name, paths in generated.items(): + manifest = write_manifest(name, paths) + print(f'{name}: {len(paths)} configs -> {manifest.relative_to(PAPER)}') + if 'baseline_fidelity' in generated: + verify_baseline_configs(generated['baseline_fidelity']) + print('baseline_fidelity config check: OK') + + +if __name__ == '__main__': + main() diff --git a/tabm_integrated/tools/run_integrated_matrix.sh b/tabm_integrated/tools/run_integrated_matrix.sh new file mode 100755 index 000000000..790f692a3 --- /dev/null +++ b/tabm_integrated/tools/run_integrated_matrix.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env bash +set -euo pipefail + +PAPER_DIR="${PAPER_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")/../paper" && pwd)}" +MANIFEST="${1:-${MANIFEST:-}}" +if [[ -z "$MANIFEST" ]]; then + echo "usage: $0 paper/exp/integrated/manifest_*.txt" >&2 + exit 2 +fi +cd "$PAPER_DIR" +if [[ ! -f "$MANIFEST" ]]; then + echo "manifest not found: $MANIFEST" >&2 + exit 2 +fi + +PYTHON="${PYTHON:-/workspace/.venvs/tabm_integrated/bin/python}" +N_GPUS="${N_GPUS:-16}" +FORCE="${FORCE:-0}" +LOG_ROOT="exp/integrated/_logs/$(basename "$MANIFEST" .txt)_$(date +%Y%m%d_%H%M%S)" +mkdir -p "$LOG_ROOT" +QUEUE="exp/integrated/_queue_$(basename "$MANIFEST" .txt)_$$.txt" +LOCK="$QUEUE.lock" +cp "$MANIFEST" "$QUEUE" +: > "$LOCK" +USAGE_CSV="$LOG_ROOT/gpu_usage.csv" +echo "gpu,config,started_at" > "$USAGE_CSV" + +cleanup_artifacts() { + local out="$1" + rm -f "$out/checkpoint.pt" "$out/checkpoint_best.pt" "$out/predictions.npz" "$out/summary.json" + rm -f "$out"/events.out.tfevents.* 2>/dev/null || true +} + +check_done() { + local out="$1" + "$PYTHON" - "$out" <<'PY' +import json, sys +from pathlib import Path +out=Path(sys.argv[1]) +report=out/'report.json' +done=out/'DONE' +if not done.exists() or not report.exists(): + raise SystemExit(1) +payload=json.loads(report.read_text()) +if payload.get('failure'): + raise SystemExit(2) +PY +} + +worker() { + local gpu="$1" + export CUDA_VISIBLE_DEVICES="$gpu" + while true; do + local cfg="" + { + flock 9 + if [[ -s "$QUEUE" ]]; then + cfg="$(head -n 1 "$QUEUE")" + tail -n +2 "$QUEUE" > "$QUEUE.tmp" + mv "$QUEUE.tmp" "$QUEUE" + fi + } 9>"$LOCK" + [[ -n "$cfg" ]] || break + local out="${cfg%.toml}" + local safe + safe="$(echo "$cfg" | tr '/ ' '__')" + local log="$LOG_ROOT/gpu${gpu}_${safe}.out" + echo "$gpu,$cfg,$(date -Is)" >> "$USAGE_CSV" + if [[ "$FORCE" != "1" ]] && check_done "$out" >/dev/null 2>&1; then + echo "SKIP $cfg" | tee -a "$log" + cleanup_artifacts "$out" + continue + fi + mkdir -p "$out" + echo "RUN gpu=$gpu cfg=$cfg out=$out" | tee "$log" + set +e + "$PYTHON" bin/run_integrated.py "$cfg" --output "$out" --force >> "$log" 2>&1 + rc=$? + set -e + cleanup_artifacts "$out" + if [[ $rc -ne 0 ]]; then + echo "FAILED rc=$rc cfg=$cfg log=$log" | tee -a "$LOG_ROOT/FAILED" + return $rc + fi + if ! check_done "$out" >/dev/null 2>&1; then + echo "FAILED missing DONE/report or failure block cfg=$cfg log=$log" | tee -a "$LOG_ROOT/FAILED" + return 1 + fi + echo "DONE $cfg" | tee -a "$log" + done +} + +pids=() +for ((gpu=0; gpu&2 + exit $rc +fi +"$PYTHON" ../tools/aggregate_integrated_results.py --stage wave --manifest "$MANIFEST" || true +used_gpus=$(tail -n +2 "$USAGE_CSV" | cut -d, -f1 | sort -n | uniq | tr '\n' ' ') +echo "used_gpus: $used_gpus" +echo "logs: $LOG_ROOT" diff --git a/tabm_integrated/tools/sberbank_combined_rescue.py b/tabm_integrated/tools/sberbank_combined_rescue.py new file mode 100644 index 000000000..d7c931a94 --- /dev/null +++ b/tabm_integrated/tools/sberbank_combined_rescue.py @@ -0,0 +1,266 @@ +from __future__ import annotations + +import argparse +import csv +import itertools +import json +import math +import shutil +import sys +import tomllib +from pathlib import Path +from typing import Any + +sys.dont_write_bytecode = True + +import tomli_w + +TOOLS = Path(__file__).resolve().parent +PAPER = TOOLS.parent / 'paper' +EXP_ROOT = PAPER / 'exp' / 'integrated' +DATASET = 'sberbank-housing' +WAVE = 'rescue_sberbank_combined' +CONFIRM_WAVE = 'rescue_sberbank_combined_confirm' +MANIFEST = EXP_ROOT / 'manifest_sberbank_combined_rescue_seed0.txt' +CONFIRM_MANIFEST = EXP_ROOT / 'manifest_sberbank_combined_rescue_final.txt' +SELECTED_PATH = PAPER / 'exp' / 'sberbank_combined_rescue_selected.csv' +FINAL_SELECTED = PAPER / 'exp' / 'selected_integrated_configs.csv' + +sys.path.insert(0, str(TOOLS)) +import aggregate_integrated_results as agg # noqa: E402 +import generate_integrated_configs as gen # noqa: E402 + + +def ftag(x: float | int) -> str: + if isinstance(x, int): + return str(x) + if x == 0: + return '0' + return f'{x:g}'.replace('-', 'm').replace('.', 'p') + + +def write_config(wave: str, variant: str, cfg: dict[str, Any], seed: int) -> Path: + cfg = json.loads(json.dumps(cfg)) + cfg['seed'] = int(seed) + if cfg.get('model', {}).get('mfb'): + cfg['model']['mfb']['mask_seed'] = int(seed) + path = EXP_ROOT / wave / DATASET / f'{variant}-evaluation' / f'{seed}.toml' + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(tomli_w.dumps(cfg)) + return path + + +def generate() -> None: + paths: list[Path] = [] + for rank, noise, lam in itertools.product([1, 2, 4], [0.0, 1e-5, 1e-4, 1e-3], [0.001, 0.005, 0.01, 0.02]): + cfg = gen.base_config(DATASET, 0) + gen.apply_rla(cfg, rank=rank, noise=noise) + gen.apply_cf_fisd(cfg, DATASET, lam=lam) + variant = f'sb_comb_rla_cf_r{rank}_n{ftag(noise)}_l{ftag(lam)}' + paths.append(write_config(WAVE, variant, cfg, 0)) + for rank, noise, rho in itertools.product([1, 2, 4], [0.0, 1e-5, 1e-4, 1e-3], [0.00025, 0.0005, 0.001, 0.0025]): + cfg = gen.base_config(DATASET, 0) + gen.apply_rla(cfg, rank=rank, noise=noise) + gen.apply_esam(cfg, rho=rho) + variant = f'sb_comb_rla_esam_r{rank}_n{ftag(noise)}_rho{ftag(rho)}' + paths.append(write_config(WAVE, variant, cfg, 0)) + for rank, noise, keep, lam in itertools.product([1, 2, 4], [1e-5, 1e-4, 1e-3], [0.9, 0.95, 0.975], [0.001, 0.005, 0.01]): + cfg = gen.base_config(DATASET, 0) + gen.apply_rla(cfg, rank=rank, noise=noise) + gen.apply_mfb(cfg, keep=keep) + gen.apply_cf_fisd(cfg, DATASET, lam=lam) + variant = f'sb_comb_rla_mfb_cf_r{rank}_n{ftag(noise)}_k{ftag(keep)}_l{ftag(lam)}' + paths.append(write_config(WAVE, variant, cfg, 0)) + MANIFEST.write_text('\n'.join(str(p.relative_to(PAPER)) for p in paths) + '\n') + print(f'{MANIFEST.relative_to(PAPER)} configs={len(paths)}') + + +def signed_delta(value: float, baseline: float, direction: str) -> tuple[float, float]: + if direction == 'lower': + d = baseline - value + return d, 100.0 * d / baseline if baseline else float('nan') + d = value - baseline + return d, 100.0 * d / abs(baseline) if baseline else float('nan') + + +def collect_candidates(wave: str, seed: int = 0) -> list[dict[str, Any]]: + rows = agg.collect_rows() + baselines = { + r['inference_mode']: r + for r in rows + if r['wave'] == 'final' + and r['dataset'] == DATASET + and r['variant'] == 'baseline_plr' + and r['seed'] == seed + and not r['failure'] + } + missing = [x for x in ['mean', 'best-head', 'greedy-heads'] if x not in baselines] + if missing: + raise RuntimeError(f'missing matched baselines for {missing}') + out: list[dict[str, Any]] = [] + for r in rows: + if r['wave'] != wave or r['dataset'] != DATASET or r['seed'] != seed or r['failure']: + continue + b = baselines[r['inference_mode']] + d, pct = signed_delta(r['validation_metric'], b['validation_metric'], r['direction']) + e = dict(r) + e['matched_validation_baseline'] = b['validation_metric'] + e['validation_delta'] = d + e['validation_percent_delta'] = pct + e['validation_status'] = 'validation_win' if d > 1e-12 else ('tie' if abs(d) <= 1e-12 else 'validation_loss') + out.append(e) + out.sort(key=lambda r: (r['validation_delta'], r['validation_score']), reverse=True) + return out + + +def write_csv(path: Path, rows: list[dict[str, Any]]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + if not rows: + path.write_text('') + return + fields: list[str] = [] + for row in rows: + for key in row: + if key not in fields: + fields.append(key) + with path.open('w', newline='') as f: + writer = csv.DictWriter(f, fieldnames=fields, lineterminator='\n') + writer.writeheader() + writer.writerows(rows) + + +def select_seed0() -> None: + candidates = collect_candidates(WAVE, 0) + top = candidates[:20] + write_csv(SELECTED_PATH, top) + winners = [r for r in candidates if r['validation_delta'] > 1e-12] + confirm = winners[:2] + paths: list[Path] = [] + for row in confirm: + cfg = tomllib.loads((PAPER / row['config_path']).read_text()) + for seed in [0, 1, 2]: + paths.append(write_config(CONFIRM_WAVE, row['variant'], cfg, seed)) + CONFIRM_MANIFEST.write_text('\n'.join(str(p.relative_to(PAPER)) for p in paths) + ('\n' if paths else '')) + print(f'{SELECTED_PATH.relative_to(PAPER)} top_rows={len(top)} validation_winners={len(winners)}') + print(f'{CONFIRM_MANIFEST.relative_to(PAPER)} configs={len(paths)}') + for row in top[:5]: + print(row['variant'], row['inference_mode'], row['validation_metric'], row['matched_validation_baseline'], row['validation_delta'], row['validation_status']) + + +def test_stats_for_variant(variant: str, inference: str) -> dict[str, Any] | None: + rows = agg.collect_rows() + baselines = [ + r for r in rows + if r['wave'] == 'final' + and r['dataset'] == DATASET + and r['variant'] == 'baseline_plr' + and r['inference_mode'] == inference + and not r['failure'] + ] + rs = [ + r for r in rows + if r['wave'] == CONFIRM_WAVE + and r['dataset'] == DATASET + and r['variant'] == variant + and r['inference_mode'] == inference + and not r['failure'] + ] + if len(rs) != 3 or len(baselines) != 3: + return None + import statistics + rs = sorted(rs, key=lambda r: r['seed']) + baselines = sorted(baselines, key=lambda r: r['seed']) + mean = statistics.mean(r['test_metric'] for r in rs) + std = statistics.stdev(r['test_metric'] for r in rs) + base_mean = statistics.mean(r['test_metric'] for r in baselines) + base_std = statistics.stdev(r['test_metric'] for r in baselines) + delta, pct = signed_delta(mean, base_mean, rs[0]['direction']) + status = agg.status_for(delta, base_std, 3, False) + return { + 'variant': variant, + 'inference_mode': inference, + 'mean': mean, + 'std': std, + 'baseline_mean': base_mean, + 'baseline_std': base_std, + 'absolute_delta': delta, + 'percent_delta': pct, + 'status': status, + } + + +def update_final_if_win() -> None: + selected = [] + if SELECTED_PATH.exists() and SELECTED_PATH.read_text().strip(): + with SELECTED_PATH.open() as f: + for row in csv.DictReader(f): + if float(row['validation_delta']) > 1e-12: + selected.append(row) + selected = selected[:2] + results = [] + chosen = None + for row in selected: + stats = test_stats_for_variant(row['variant'], row['inference_mode']) + if stats is None: + continue + result = dict(row) | stats + results.append(result) + if chosen is None and stats['status'] in {'clear_win', 'weak_win'}: + chosen = result + write_csv(PAPER / 'exp' / 'sberbank_combined_rescue_confirmed.csv', results) + if chosen is None: + print('no_confirmed_win') + for r in results: + print(r['variant'], r['inference_mode'], r['absolute_delta'], r['status']) + return + + variant = chosen['variant'] + # Copy confirmation outputs into final best_combined slots. + for seed in [0, 1, 2]: + src_cfg = EXP_ROOT / CONFIRM_WAVE / DATASET / f'{variant}-evaluation' / f'{seed}.toml' + dst_cfg = EXP_ROOT / 'final' / DATASET / 'best_combined-evaluation' / f'{seed}.toml' + dst_cfg.write_text(src_cfg.read_text()) + for suffix in ['-evaluation', '-best-head-evaluation', '-greedy-heads-evaluation']: + src = EXP_ROOT / CONFIRM_WAVE / DATASET / f'{variant}{suffix}' / str(seed) + dst = EXP_ROOT / 'final' / DATASET / f'best_combined{suffix}' / str(seed) + if src.exists(): + if dst.exists(): + shutil.rmtree(dst) + dst.parent.mkdir(parents=True, exist_ok=True) + shutil.copytree(src, dst) + # Update validation-selected configs. + rows = [] + with FINAL_SELECTED.open() as f: + for row in csv.DictReader(f): + if row['dataset'] == DATASET and row['final_variant'] == 'best_combined': + row.update({ + 'source_variant': chosen['variant'], + 'source_wave': WAVE, + 'inference_mode': chosen['inference_mode'], + 'validation_metric': chosen['validation_metric'], + 'validation_score': chosen['validation_score'], + 'matched_validation_baseline': str(chosen['matched_validation_baseline']), + 'validation_delta': str(chosen['validation_delta']), + 'validation_percent_delta': str(chosen['validation_percent_delta']), + 'validation_status': chosen['validation_status'], + 'source_config_path': chosen['config_path'], + }) + rows.append(row) + write_csv(FINAL_SELECTED, rows) + print('updated_final_best_combined', chosen['variant'], chosen['inference_mode'], chosen['absolute_delta'], chosen['status']) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument('stage', choices=['generate', 'select-seed0', 'update-final-if-win']) + args = parser.parse_args() + if args.stage == 'generate': + generate() + elif args.stage == 'select-seed0': + select_seed0() + elif args.stage == 'update-final-if-win': + update_final_if_win() + + +if __name__ == '__main__': + main()